2024-11-13 16:25:55,729 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 16:25:55,748 main DEBUG Took 0.016429 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 16:25:55,749 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 16:25:55,750 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 16:25:55,751 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 16:25:55,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,764 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 16:25:55,783 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,786 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,787 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,789 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,790 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,790 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,792 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,793 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,794 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,794 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,795 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,795 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,797 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,799 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,801 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,803 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 16:25:55,804 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,805 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 16:25:55,807 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 16:25:55,810 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 16:25:55,813 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 16:25:55,813 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 16:25:55,816 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 16:25:55,816 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 16:25:55,830 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 16:25:55,833 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 16:25:55,840 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 16:25:55,840 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 16:25:55,841 main DEBUG createAppenders(={Console}) 2024-11-13 16:25:55,843 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-13 16:25:55,843 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 16:25:55,844 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-13 16:25:55,845 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 16:25:55,845 main DEBUG OutputStream closed 2024-11-13 16:25:55,846 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 16:25:55,848 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 16:25:55,848 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-13 16:25:55,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 16:25:55,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 16:25:55,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 16:25:55,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 16:25:55,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 16:25:55,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 16:25:55,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 16:25:55,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 16:25:55,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 16:25:55,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 16:25:55,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 16:25:55,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 16:25:55,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 16:25:55,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 16:25:55,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 16:25:55,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 16:25:55,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 16:25:55,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 16:25:55,978 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 16:25:55,978 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-13 16:25:55,979 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 16:25:55,979 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-13T16:25:56,301 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208 2024-11-13 16:25:56,305 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 16:25:56,306 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T16:25:56,319 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T16:25:56,326 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-11-13T16:25:56,351 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T16:25:56,405 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T16:25:56,405 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T16:25:56,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T16:25:56,439 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066, deleteOnExit=true 2024-11-13T16:25:56,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T16:25:56,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/test.cache.data in system properties and HBase conf 2024-11-13T16:25:56,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T16:25:56,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir in system properties and HBase conf 2024-11-13T16:25:56,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T16:25:56,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T16:25:56,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T16:25:56,522 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T16:25:56,642 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T16:25:56,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T16:25:56,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T16:25:56,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T16:25:56,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T16:25:56,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T16:25:56,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T16:25:56,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T16:25:56,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T16:25:56,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T16:25:56,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/nfs.dump.dir in system properties and HBase conf 2024-11-13T16:25:56,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/java.io.tmpdir in system properties and HBase conf 2024-11-13T16:25:56,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T16:25:56,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T16:25:56,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T16:25:58,015 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T16:25:58,121 INFO [Time-limited test {}] log.Log(170): Logging initialized @3387ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T16:25:58,227 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:25:58,336 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T16:25:58,419 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T16:25:58,419 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T16:25:58,421 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T16:25:58,447 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:25:58,454 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir/,AVAILABLE} 2024-11-13T16:25:58,455 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T16:25:58,691 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/java.io.tmpdir/jetty-localhost-39229-hadoop-hdfs-3_4_1-tests_jar-_-any-14183059296891330070/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T16:25:58,701 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:39229} 2024-11-13T16:25:58,701 INFO [Time-limited test {}] server.Server(415): Started @3968ms 2024-11-13T16:25:59,388 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:25:59,398 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T16:25:59,402 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T16:25:59,402 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T16:25:59,402 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T16:25:59,404 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir/,AVAILABLE} 2024-11-13T16:25:59,405 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T16:25:59,539 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/java.io.tmpdir/jetty-localhost-33127-hadoop-hdfs-3_4_1-tests_jar-_-any-5164183008394974492/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:25:59,540 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:33127} 2024-11-13T16:25:59,541 INFO [Time-limited test {}] server.Server(415): Started @4807ms 2024-11-13T16:25:59,611 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T16:25:59,799 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:25:59,809 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T16:25:59,827 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T16:25:59,827 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T16:25:59,828 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T16:25:59,834 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir/,AVAILABLE} 2024-11-13T16:25:59,835 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T16:25:59,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/java.io.tmpdir/jetty-localhost-38879-hadoop-hdfs-3_4_1-tests_jar-_-any-2892287773022963851/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:25:59,977 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:38879} 2024-11-13T16:25:59,977 INFO [Time-limited test {}] server.Server(415): Started @5243ms 2024-11-13T16:25:59,981 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T16:26:00,048 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T16:26:00,057 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T16:26:00,064 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T16:26:00,065 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T16:26:00,065 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T16:26:00,067 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir/,AVAILABLE} 2024-11-13T16:26:00,068 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T16:26:00,188 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/java.io.tmpdir/jetty-localhost-36057-hadoop-hdfs-3_4_1-tests_jar-_-any-5671763251007727880/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:26:00,189 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:36057} 2024-11-13T16:26:00,189 INFO [Time-limited test {}] server.Server(415): Started @5456ms 2024-11-13T16:26:00,191 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T16:26:00,886 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data1/current/BP-572455086-172.17.0.3-1731515157445/current, will proceed with Du for space computation calculation, 2024-11-13T16:26:00,887 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data2/current/BP-572455086-172.17.0.3-1731515157445/current, will proceed with Du for space computation calculation, 2024-11-13T16:26:00,888 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data3/current/BP-572455086-172.17.0.3-1731515157445/current, will proceed with Du for space computation calculation, 2024-11-13T16:26:00,895 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data4/current/BP-572455086-172.17.0.3-1731515157445/current, will proceed with Du for space computation calculation, 2024-11-13T16:26:00,946 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T16:26:00,951 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T16:26:00,988 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data6/current/BP-572455086-172.17.0.3-1731515157445/current, will proceed with Du for space computation calculation, 2024-11-13T16:26:00,991 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data5/current/BP-572455086-172.17.0.3-1731515157445/current, will proceed with Du for space computation calculation, 2024-11-13T16:26:01,051 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T16:26:01,062 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb5aaedbe9b329b4 with lease ID 0xc8e8335e77d6f006: Processing first storage report for DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9 from datanode DatanodeRegistration(127.0.0.1:36473, datanodeUuid=2db7fc52-8d40-4da7-b13b-25d17df12739, infoPort=46565, infoSecurePort=0, ipcPort=39039, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445) 2024-11-13T16:26:01,065 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb5aaedbe9b329b4 with lease ID 0xc8e8335e77d6f006: from storage DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9 node DatanodeRegistration(127.0.0.1:36473, datanodeUuid=2db7fc52-8d40-4da7-b13b-25d17df12739, infoPort=46565, infoSecurePort=0, ipcPort=39039, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-13T16:26:01,066 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x122fc4f872af45d2 with lease ID 0xc8e8335e77d6f005: Processing first storage report for DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1 from datanode DatanodeRegistration(127.0.0.1:33305, datanodeUuid=7adbdbbb-b812-4222-a2c5-0af984f390b0, infoPort=44125, infoSecurePort=0, ipcPort=45141, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445) 2024-11-13T16:26:01,066 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x122fc4f872af45d2 with lease ID 0xc8e8335e77d6f005: from storage DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1 node DatanodeRegistration(127.0.0.1:33305, datanodeUuid=7adbdbbb-b812-4222-a2c5-0af984f390b0, infoPort=44125, infoSecurePort=0, ipcPort=45141, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:26:01,066 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb5aaedbe9b329b4 with lease ID 0xc8e8335e77d6f006: Processing first storage report for DS-d74486dc-2963-47c4-bede-b28c6b872ad2 from datanode DatanodeRegistration(127.0.0.1:36473, datanodeUuid=2db7fc52-8d40-4da7-b13b-25d17df12739, infoPort=46565, infoSecurePort=0, ipcPort=39039, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445) 2024-11-13T16:26:01,067 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb5aaedbe9b329b4 with lease ID 0xc8e8335e77d6f006: from storage DS-d74486dc-2963-47c4-bede-b28c6b872ad2 node DatanodeRegistration(127.0.0.1:36473, datanodeUuid=2db7fc52-8d40-4da7-b13b-25d17df12739, infoPort=46565, infoSecurePort=0, ipcPort=39039, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T16:26:01,069 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x122fc4f872af45d2 with lease ID 0xc8e8335e77d6f005: Processing first storage report for DS-38fefe66-b906-4dfa-9ad4-29b84b0824ac from datanode DatanodeRegistration(127.0.0.1:33305, datanodeUuid=7adbdbbb-b812-4222-a2c5-0af984f390b0, infoPort=44125, infoSecurePort=0, ipcPort=45141, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445) 2024-11-13T16:26:01,069 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x122fc4f872af45d2 with lease ID 0xc8e8335e77d6f005: from storage DS-38fefe66-b906-4dfa-9ad4-29b84b0824ac node DatanodeRegistration(127.0.0.1:33305, datanodeUuid=7adbdbbb-b812-4222-a2c5-0af984f390b0, infoPort=44125, infoSecurePort=0, ipcPort=45141, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:26:01,072 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d6f4487fddc6916 with lease ID 0xc8e8335e77d6f007: Processing first storage report for DS-5cadb267-b993-475f-b1b9-324c447cbca9 from datanode DatanodeRegistration(127.0.0.1:33543, datanodeUuid=7942aa5b-900b-4db3-b69a-89950f743638, infoPort=42379, infoSecurePort=0, ipcPort=33903, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445) 2024-11-13T16:26:01,072 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d6f4487fddc6916 with lease ID 0xc8e8335e77d6f007: from storage DS-5cadb267-b993-475f-b1b9-324c447cbca9 node DatanodeRegistration(127.0.0.1:33543, datanodeUuid=7942aa5b-900b-4db3-b69a-89950f743638, infoPort=42379, infoSecurePort=0, ipcPort=33903, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:26:01,073 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d6f4487fddc6916 with lease ID 0xc8e8335e77d6f007: Processing first storage report for DS-08d4a9a6-0a45-46f5-a09d-72a979a9f47b from datanode DatanodeRegistration(127.0.0.1:33543, datanodeUuid=7942aa5b-900b-4db3-b69a-89950f743638, infoPort=42379, infoSecurePort=0, ipcPort=33903, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445) 2024-11-13T16:26:01,073 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d6f4487fddc6916 with lease ID 0xc8e8335e77d6f007: from storage DS-08d4a9a6-0a45-46f5-a09d-72a979a9f47b node DatanodeRegistration(127.0.0.1:33543, datanodeUuid=7942aa5b-900b-4db3-b69a-89950f743638, infoPort=42379, infoSecurePort=0, ipcPort=33903, storageInfo=lv=-57;cid=testClusterID;nsid=203995599;c=1731515157445), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T16:26:01,100 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208 2024-11-13T16:26:01,259 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/zookeeper_0, clientPort=54878, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T16:26:01,308 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54878 2024-11-13T16:26:01,335 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:01,346 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:01,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741825_1001 (size=7) 2024-11-13T16:26:01,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741825_1001 (size=7) 2024-11-13T16:26:01,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741825_1001 (size=7) 2024-11-13T16:26:02,046 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19 with version=8 2024-11-13T16:26:02,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/hbase-staging 2024-11-13T16:26:02,403 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f8f85b0bb1d8:0 server-side Connection retries=45 2024-11-13T16:26:02,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:02,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:02,426 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T16:26:02,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:02,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T16:26:02,603 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T16:26:02,692 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T16:26:02,707 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T16:26:02,713 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T16:26:02,753 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 44132 (auto-detected) 2024-11-13T16:26:02,754 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T16:26:02,786 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34015 2024-11-13T16:26:02,813 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34015 connecting to ZooKeeper ensemble=127.0.0.1:54878 2024-11-13T16:26:02,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:340150x0, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T16:26:02,916 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34015-0x101363260180000 connected 2024-11-13T16:26:03,031 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,035 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,050 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:26:03,054 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19, hbase.cluster.distributed=false 2024-11-13T16:26:03,087 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T16:26:03,092 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34015 2024-11-13T16:26:03,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34015 2024-11-13T16:26:03,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34015 2024-11-13T16:26:03,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34015 2024-11-13T16:26:03,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34015 2024-11-13T16:26:03,246 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f8f85b0bb1d8:0 server-side Connection retries=45 2024-11-13T16:26:03,248 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,249 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T16:26:03,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T16:26:03,253 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T16:26:03,257 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T16:26:03,258 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37847 2024-11-13T16:26:03,261 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37847 connecting to ZooKeeper ensemble=127.0.0.1:54878 2024-11-13T16:26:03,263 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,267 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378470x0, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T16:26:03,286 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378470x0, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:26:03,286 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37847-0x101363260180001 connected 2024-11-13T16:26:03,291 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T16:26:03,302 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T16:26:03,307 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T16:26:03,315 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T16:26:03,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37847 2024-11-13T16:26:03,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37847 2024-11-13T16:26:03,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37847 2024-11-13T16:26:03,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37847 2024-11-13T16:26:03,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37847 2024-11-13T16:26:03,360 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f8f85b0bb1d8:0 server-side Connection retries=45 2024-11-13T16:26:03,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,361 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T16:26:03,361 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,362 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T16:26:03,362 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T16:26:03,362 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T16:26:03,364 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33401 2024-11-13T16:26:03,367 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33401 connecting to ZooKeeper ensemble=127.0.0.1:54878 2024-11-13T16:26:03,371 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,376 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334010x0, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T16:26:03,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33401-0x101363260180002 connected 2024-11-13T16:26:03,436 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:26:03,437 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T16:26:03,445 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T16:26:03,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T16:26:03,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T16:26:03,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33401 2024-11-13T16:26:03,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33401 2024-11-13T16:26:03,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33401 2024-11-13T16:26:03,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33401 2024-11-13T16:26:03,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33401 2024-11-13T16:26:03,498 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f8f85b0bb1d8:0 server-side Connection retries=45 2024-11-13T16:26:03,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,499 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T16:26:03,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T16:26:03,500 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T16:26:03,500 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T16:26:03,501 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T16:26:03,504 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40565 2024-11-13T16:26:03,508 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40565 connecting to ZooKeeper ensemble=127.0.0.1:54878 2024-11-13T16:26:03,511 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,515 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405650x0, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T16:26:03,544 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:405650x0, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:26:03,545 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T16:26:03,548 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40565-0x101363260180003 connected 2024-11-13T16:26:03,561 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T16:26:03,568 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T16:26:03,587 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T16:26:03,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40565 2024-11-13T16:26:03,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40565 2024-11-13T16:26:03,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40565 2024-11-13T16:26:03,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40565 2024-11-13T16:26:03,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40565 2024-11-13T16:26:03,659 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f8f85b0bb1d8:34015 2024-11-13T16:26:03,661 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:03,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:26:03,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:26:03,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:26:03,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:26:03,689 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:03,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T16:26:03,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T16:26:03,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T16:26:03,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:03,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:03,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:03,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:03,736 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T16:26:03,738 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f8f85b0bb1d8,34015,1731515162192 from backup master directory 2024-11-13T16:26:03,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:03,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:26:03,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:26:03,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:26:03,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T16:26:03,752 WARN [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T16:26:03,752 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:03,755 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T16:26:03,757 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T16:26:03,839 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/hbase.id] with ID: f6d1291a-e193-41b7-8dec-6caf412160f7 2024-11-13T16:26:03,839 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/.tmp/hbase.id 2024-11-13T16:26:03,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741826_1002 (size=42) 2024-11-13T16:26:03,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741826_1002 (size=42) 2024-11-13T16:26:03,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741826_1002 (size=42) 2024-11-13T16:26:03,856 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/.tmp/hbase.id]:[hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/hbase.id] 2024-11-13T16:26:03,905 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T16:26:03,910 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T16:26:03,931 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-13T16:26:03,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:03,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:03,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:03,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:03,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741827_1003 (size=196) 2024-11-13T16:26:03,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741827_1003 (size=196) 2024-11-13T16:26:03,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741827_1003 (size=196) 2024-11-13T16:26:03,981 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T16:26:03,983 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T16:26:03,988 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:04,015 WARN [IPC Server handler 3 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:04,015 WARN [IPC Server handler 3 on default port 40955 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T16:26:04,015 WARN [IPC Server handler 3 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T16:26:04,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741828_1004 (size=1189) 2024-11-13T16:26:04,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741828_1004 (size=1189) 2024-11-13T16:26:04,042 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store 2024-11-13T16:26:04,064 WARN [IPC Server handler 4 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:04,065 WARN [IPC Server handler 4 on default port 40955 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T16:26:04,065 WARN [IPC Server handler 4 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T16:26:04,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741829_1005 (size=34) 2024-11-13T16:26:04,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741829_1005 (size=34) 2024-11-13T16:26:04,088 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T16:26:04,092 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:04,093 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T16:26:04,094 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:26:04,094 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:26:04,096 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T16:26:04,096 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:26:04,097 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:26:04,098 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731515164093Disabling compacts and flushes for region at 1731515164093Disabling writes for close at 1731515164096 (+3 ms)Writing region close event to WAL at 1731515164097 (+1 ms)Closed at 1731515164097 2024-11-13T16:26:04,102 WARN [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/.initializing 2024-11-13T16:26:04,102 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/WALs/f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:04,113 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:04,135 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C34015%2C1731515162192, suffix=, logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/WALs/f8f85b0bb1d8,34015,1731515162192, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/oldWALs, maxLogs=10 2024-11-13T16:26:04,182 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/WALs/f8f85b0bb1d8,34015,1731515162192/f8f85b0bb1d8%2C34015%2C1731515162192.1731515164142, exclude list is [], retry=0 2024-11-13T16:26:04,195 WARN [IPC Server handler 2 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:04,195 WARN [IPC Server handler 2 on default port 40955 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T16:26:04,196 WARN [IPC Server handler 2 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T16:26:04,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:04,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:04,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:04,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T16:26:04,304 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/WALs/f8f85b0bb1d8,34015,1731515162192/f8f85b0bb1d8%2C34015%2C1731515162192.1731515164142 2024-11-13T16:26:04,308 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:04,309 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:04,310 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:04,316 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,318 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,381 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T16:26:04,437 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:04,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:04,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T16:26:04,448 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:04,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:04,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T16:26:04,457 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:04,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:04,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,468 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T16:26:04,469 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:04,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:04,474 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,480 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,486 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,504 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,504 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,515 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T16:26:04,522 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T16:26:04,535 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:04,543 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68790134, jitterRate=0.02505287528038025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T16:26:04,555 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731515164341Initializing all the Stores at 1731515164343 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515164344 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515164346 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515164346Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515164346Cleaning up temporary data from old regions at 1731515164505 (+159 ms)Region opened successfully at 1731515164555 (+50 ms) 2024-11-13T16:26:04,559 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T16:26:04,628 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67fbb0f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f8f85b0bb1d8/172.17.0.3:0 2024-11-13T16:26:04,686 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T16:26:04,706 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T16:26:04,706 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T16:26:04,712 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T16:26:04,715 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-13T16:26:04,724 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-13T16:26:04,725 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T16:26:04,799 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T16:26:04,825 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T16:26:04,841 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T16:26:04,846 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T16:26:04,851 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T16:26:04,858 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T16:26:04,862 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T16:26:04,873 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T16:26:04,890 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T16:26:04,897 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T16:26:04,916 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T16:26:04,963 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T16:26:04,975 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T16:26:04,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T16:26:04,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T16:26:04,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T16:26:04,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:04,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T16:26:04,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:04,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:04,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:04,996 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f8f85b0bb1d8,34015,1731515162192, sessionid=0x101363260180000, setting cluster-up flag (Was=false) 2024-11-13T16:26:05,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:05,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:05,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:05,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:05,063 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T16:26:05,066 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:05,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:05,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:05,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:05,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:05,166 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T16:26:05,169 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:05,180 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T16:26:05,213 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:05,213 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:05,214 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T16:26:05,214 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T16:26:05,214 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:05,214 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:05,214 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T16:26:05,214 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T16:26:05,245 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(746): ClusterId : f6d1291a-e193-41b7-8dec-6caf412160f7 2024-11-13T16:26:05,245 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(746): ClusterId : f6d1291a-e193-41b7-8dec-6caf412160f7 2024-11-13T16:26:05,245 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(746): ClusterId : f6d1291a-e193-41b7-8dec-6caf412160f7 2024-11-13T16:26:05,249 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T16:26:05,249 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T16:26:05,251 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T16:26:05,272 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T16:26:05,273 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T16:26:05,283 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T16:26:05,283 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T16:26:05,284 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T16:26:05,284 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T16:26:05,291 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T16:26:05,305 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T16:26:05,305 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T16:26:05,306 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b26aca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f8f85b0bb1d8/172.17.0.3:0 2024-11-13T16:26:05,307 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T16:26:05,308 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad2ee9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f8f85b0bb1d8/172.17.0.3:0 2024-11-13T16:26:05,309 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T16:26:05,310 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6858e46e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f8f85b0bb1d8/172.17.0.3:0 2024-11-13T16:26:05,320 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T16:26:05,325 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f8f85b0bb1d8:40565 2024-11-13T16:26:05,326 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f8f85b0bb1d8:37847 2024-11-13T16:26:05,327 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f8f85b0bb1d8:33401 2024-11-13T16:26:05,329 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T16:26:05,329 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T16:26:05,329 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T16:26:05,330 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T16:26:05,330 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T16:26:05,330 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T16:26:05,328 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f8f85b0bb1d8,34015,1731515162192 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T16:26:05,329 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T16:26:05,334 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T16:26:05,334 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T16:26:05,337 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,34015,1731515162192 with port=40565, startcode=1731515163497 2024-11-13T16:26:05,337 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,34015,1731515162192 with port=37847, startcode=1731515163204 2024-11-13T16:26:05,339 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,34015,1731515162192 with port=33401, startcode=1731515163358 2024-11-13T16:26:05,343 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f8f85b0bb1d8:0, corePoolSize=5, maxPoolSize=5 2024-11-13T16:26:05,344 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f8f85b0bb1d8:0, corePoolSize=5, maxPoolSize=5 2024-11-13T16:26:05,344 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=5, maxPoolSize=5 2024-11-13T16:26:05,344 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=5, maxPoolSize=5 2024-11-13T16:26:05,344 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f8f85b0bb1d8:0, corePoolSize=10, maxPoolSize=10 2024-11-13T16:26:05,345 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,345 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=2, maxPoolSize=2 2024-11-13T16:26:05,345 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,353 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T16:26:05,354 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T16:26:05,354 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T16:26:05,379 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731515195379 2024-11-13T16:26:05,382 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T16:26:05,384 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T16:26:05,388 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T16:26:05,388 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T16:26:05,389 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T16:26:05,389 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T16:26:05,399 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T16:26:05,400 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T16:26:05,396 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,406 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T16:26:05,408 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T16:26:05,408 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T16:26:05,418 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:05,419 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T16:26:05,420 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T16:26:05,420 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T16:26:05,425 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33895, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T16:26:05,425 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58933, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T16:26:05,425 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59357, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T16:26:05,428 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.large.0-1731515165425,5,FailOnTimeoutGroup] 2024-11-13T16:26:05,432 WARN [IPC Server handler 3 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:05,432 WARN [IPC Server handler 3 on default port 40955 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T16:26:05,433 WARN [IPC Server handler 3 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T16:26:05,437 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:26:05,451 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:26:05,457 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T16:26:05,459 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.small.0-1731515165428,5,FailOnTimeoutGroup] 2024-11-13T16:26:05,459 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,459 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T16:26:05,460 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,460 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,482 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T16:26:05,482 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T16:26:05,482 WARN [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T16:26:05,482 WARN [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T16:26:05,482 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T16:26:05,483 WARN [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T16:26:05,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741831_1007 (size=1321) 2024-11-13T16:26:05,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741831_1007 (size=1321) 2024-11-13T16:26:05,501 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T16:26:05,501 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19 2024-11-13T16:26:05,508 WARN [IPC Server handler 1 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:05,508 WARN [IPC Server handler 1 on default port 40955 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T16:26:05,508 WARN [IPC Server handler 1 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T16:26:05,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741832_1008 (size=32) 2024-11-13T16:26:05,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741832_1008 (size=32) 2024-11-13T16:26:05,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:05,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T16:26:05,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T16:26:05,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:05,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:05,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T16:26:05,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T16:26:05,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:05,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:05,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T16:26:05,534 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T16:26:05,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:05,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:05,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T16:26:05,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T16:26:05,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:05,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:05,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T16:26:05,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740 2024-11-13T16:26:05,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740 2024-11-13T16:26:05,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T16:26:05,548 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T16:26:05,549 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T16:26:05,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T16:26:05,558 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:05,559 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59350910, jitterRate=-0.1156025230884552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T16:26:05,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731515165520Initializing all the Stores at 1731515165522 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515165522Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515165523 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515165523Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515165523Cleaning up temporary data from old regions at 1731515165548 (+25 ms)Region opened successfully at 1731515165564 (+16 ms) 2024-11-13T16:26:05,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T16:26:05,565 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T16:26:05,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T16:26:05,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T16:26:05,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T16:26:05,567 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T16:26:05,567 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731515165565Disabling compacts and flushes for region at 1731515165565Disabling writes for close at 1731515165565Writing region close event to WAL at 1731515165566 (+1 ms)Closed at 1731515165567 (+1 ms) 2024-11-13T16:26:05,571 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T16:26:05,571 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T16:26:05,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T16:26:05,583 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,34015,1731515162192 with port=33401, startcode=1731515163358 2024-11-13T16:26:05,584 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,34015,1731515162192 with port=40565, startcode=1731515163497 2024-11-13T16:26:05,584 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(2659): reportForDuty to master=f8f85b0bb1d8,34015,1731515162192 with port=37847, startcode=1731515163204 2024-11-13T16:26:05,586 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:05,589 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] master.ServerManager(517): Registering regionserver=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:05,589 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:26:05,593 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T16:26:05,601 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:05,601 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19 2024-11-13T16:26:05,601 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] master.ServerManager(517): Registering regionserver=f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:05,601 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40955 2024-11-13T16:26:05,601 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T16:26:05,605 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:05,606 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19 2024-11-13T16:26:05,606 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] master.ServerManager(517): Registering regionserver=f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:05,606 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40955 2024-11-13T16:26:05,607 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T16:26:05,614 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19 2024-11-13T16:26:05,614 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40955 2024-11-13T16:26:05,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T16:26:05,614 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T16:26:05,642 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] zookeeper.ZKUtil(111): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:05,642 WARN [RS:1;f8f85b0bb1d8:33401 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T16:26:05,642 INFO [RS:1;f8f85b0bb1d8:33401 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:05,643 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:05,658 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] zookeeper.ZKUtil(111): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:05,659 WARN [RS:0;f8f85b0bb1d8:37847 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T16:26:05,659 INFO [RS:0;f8f85b0bb1d8:37847 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:05,659 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:05,661 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] zookeeper.ZKUtil(111): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:05,661 WARN [RS:2;f8f85b0bb1d8:40565 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T16:26:05,662 INFO [RS:2;f8f85b0bb1d8:40565 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:05,662 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:05,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f8f85b0bb1d8,40565,1731515163497] 2024-11-13T16:26:05,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f8f85b0bb1d8,37847,1731515163204] 2024-11-13T16:26:05,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f8f85b0bb1d8,33401,1731515163358] 2024-11-13T16:26:05,692 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T16:26:05,692 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T16:26:05,705 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T16:26:05,717 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T16:26:05,723 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T16:26:05,723 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T16:26:05,730 INFO [RS:2;f8f85b0bb1d8:40565 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T16:26:05,730 INFO [RS:1;f8f85b0bb1d8:33401 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T16:26:05,731 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,731 INFO [RS:0;f8f85b0bb1d8:37847 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T16:26:05,731 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,731 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,738 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T16:26:05,738 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T16:26:05,741 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T16:26:05,744 WARN [f8f85b0bb1d8:34015 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T16:26:05,747 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T16:26:05,749 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T16:26:05,750 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,750 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,750 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,750 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,750 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,750 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,751 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,751 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,751 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T16:26:05,751 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=2, maxPoolSize=2 2024-11-13T16:26:05,751 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,751 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,751 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,751 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,752 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,752 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,752 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,752 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,752 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,752 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,752 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=2, maxPoolSize=2 2024-11-13T16:26:05,752 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,752 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,753 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,753 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,753 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,753 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,753 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:26:05,753 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:26:05,754 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,754 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,754 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,754 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=2, maxPoolSize=2 2024-11-13T16:26:05,754 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,757 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,758 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,758 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,758 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,758 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,758 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,759 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,759 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f8f85b0bb1d8:0, corePoolSize=1, maxPoolSize=1 2024-11-13T16:26:05,759 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:26:05,759 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:26:05,759 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:26:05,759 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f8f85b0bb1d8:0, corePoolSize=3, maxPoolSize=3 2024-11-13T16:26:05,764 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,764 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,764 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,764 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,764 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,764 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,765 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,765 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,765 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,765 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,37847,1731515163204-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T16:26:05,765 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,765 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,40565,1731515163497-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T16:26:05,771 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,771 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,771 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,771 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,771 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,772 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,33401,1731515163358-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T16:26:05,804 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T16:26:05,804 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T16:26:05,804 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T16:26:05,807 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,37847,1731515163204-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,807 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,33401,1731515163358-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,807 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,40565,1731515163497-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,807 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,807 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,807 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.Replication(171): f8f85b0bb1d8,33401,1731515163358 started 2024-11-13T16:26:05,807 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.Replication(171): f8f85b0bb1d8,40565,1731515163497 started 2024-11-13T16:26:05,807 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,808 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.Replication(171): f8f85b0bb1d8,37847,1731515163204 started 2024-11-13T16:26:05,831 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,831 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,832 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1482): Serving as f8f85b0bb1d8,37847,1731515163204, RpcServer on f8f85b0bb1d8/172.17.0.3:37847, sessionid=0x101363260180001 2024-11-13T16:26:05,832 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1482): Serving as f8f85b0bb1d8,40565,1731515163497, RpcServer on f8f85b0bb1d8/172.17.0.3:40565, sessionid=0x101363260180003 2024-11-13T16:26:05,833 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T16:26:05,833 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T16:26:05,833 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:05,833 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:05,833 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:05,833 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,37847,1731515163204' 2024-11-13T16:26:05,833 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,40565,1731515163497' 2024-11-13T16:26:05,833 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1482): Serving as f8f85b0bb1d8,33401,1731515163358, RpcServer on f8f85b0bb1d8/172.17.0.3:33401, sessionid=0x101363260180002 2024-11-13T16:26:05,833 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T16:26:05,833 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T16:26:05,834 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T16:26:05,834 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:05,834 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,33401,1731515163358' 2024-11-13T16:26:05,834 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T16:26:05,834 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T16:26:05,835 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T16:26:05,836 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T16:26:05,836 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T16:26:05,836 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:05,836 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,40565,1731515163497' 2024-11-13T16:26:05,836 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T16:26:05,839 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T16:26:05,839 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T16:26:05,839 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T16:26:05,839 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:05,839 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,37847,1731515163204' 2024-11-13T16:26:05,839 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T16:26:05,842 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T16:26:05,842 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T16:26:05,842 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T16:26:05,842 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T16:26:05,842 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:05,842 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f8f85b0bb1d8,33401,1731515163358' 2024-11-13T16:26:05,842 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T16:26:05,843 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T16:26:05,843 INFO [RS:2;f8f85b0bb1d8:40565 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T16:26:05,843 INFO [RS:2;f8f85b0bb1d8:40565 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T16:26:05,843 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T16:26:05,843 INFO [RS:0;f8f85b0bb1d8:37847 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T16:26:05,843 INFO [RS:0;f8f85b0bb1d8:37847 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T16:26:05,844 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T16:26:05,845 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T16:26:05,846 INFO [RS:1;f8f85b0bb1d8:33401 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T16:26:05,846 INFO [RS:1;f8f85b0bb1d8:33401 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T16:26:05,950 INFO [RS:1;f8f85b0bb1d8:33401 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:05,950 INFO [RS:2;f8f85b0bb1d8:40565 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:05,950 INFO [RS:0;f8f85b0bb1d8:37847 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:05,955 INFO [RS:0;f8f85b0bb1d8:37847 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C37847%2C1731515163204, suffix=, logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,37847,1731515163204, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs, maxLogs=32 2024-11-13T16:26:05,955 INFO [RS:1;f8f85b0bb1d8:33401 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C33401%2C1731515163358, suffix=, logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs, maxLogs=32 2024-11-13T16:26:05,955 INFO [RS:2;f8f85b0bb1d8:40565 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C40565%2C1731515163497, suffix=, logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,40565,1731515163497, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs, maxLogs=32 2024-11-13T16:26:05,989 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, exclude list is [], retry=0 2024-11-13T16:26:05,990 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,40565,1731515163497/f8f85b0bb1d8%2C40565%2C1731515163497.1731515165964, exclude list is [], retry=0 2024-11-13T16:26:05,992 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,37847,1731515163204/f8f85b0bb1d8%2C37847%2C1731515163204.1731515165968, exclude list is [], retry=0 2024-11-13T16:26:05,997 WARN [IPC Server handler 4 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-13T16:26:05,997 WARN [IPC Server handler 4 on default port 40955 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-13T16:26:05,997 WARN [IPC Server handler 4 on default port 40955 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-13T16:26:05,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:06,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:06,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:06,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:06,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:06,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:06,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:06,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:06,079 INFO [RS:1;f8f85b0bb1d8:33401 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963 2024-11-13T16:26:06,083 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:06,085 INFO [RS:2;f8f85b0bb1d8:40565 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,40565,1731515163497/f8f85b0bb1d8%2C40565%2C1731515163497.1731515165964 2024-11-13T16:26:06,089 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:06,120 INFO [RS:0;f8f85b0bb1d8:37847 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,37847,1731515163204/f8f85b0bb1d8%2C37847%2C1731515163204.1731515165968 2024-11-13T16:26:06,122 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:06,248 DEBUG [f8f85b0bb1d8:34015 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T16:26:06,261 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(204): Hosts are {f8f85b0bb1d8=0} racks are {/default-rack=0} 2024-11-13T16:26:06,274 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T16:26:06,274 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T16:26:06,274 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T16:26:06,274 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T16:26:06,274 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T16:26:06,274 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T16:26:06,274 INFO [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T16:26:06,274 INFO [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T16:26:06,274 INFO [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T16:26:06,274 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T16:26:06,288 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:06,296 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,33401,1731515163358, state=OPENING 2024-11-13T16:26:06,313 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T16:26:06,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:06,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:06,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:06,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:06,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:06,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:06,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:06,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:06,325 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:26:06,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,33401,1731515163358}] 2024-11-13T16:26:06,522 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T16:26:06,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55969, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:26:06,557 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T16:26:06,558 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:06,559 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T16:26:06,565 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C33401%2C1731515163358.meta, suffix=.meta, logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs, maxLogs=32 2024-11-13T16:26:06,592 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, exclude list is [], retry=0 2024-11-13T16:26:06,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:06,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:06,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:06,644 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta 2024-11-13T16:26:06,647 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:06,647 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:06,649 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T16:26:06,653 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T16:26:06,671 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T16:26:06,677 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T16:26:06,678 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:06,679 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T16:26:06,679 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T16:26:06,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T16:26:06,701 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T16:26:06,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:06,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:06,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T16:26:06,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T16:26:06,716 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:06,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:06,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T16:26:06,726 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T16:26:06,727 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:06,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:06,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T16:26:06,734 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T16:26:06,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:06,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:06,738 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T16:26:06,743 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740 2024-11-13T16:26:06,758 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740 2024-11-13T16:26:06,779 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T16:26:06,779 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T16:26:06,782 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T16:26:06,800 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T16:26:06,808 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72782686, jitterRate=0.0845465362071991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T16:26:06,808 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T16:26:06,810 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731515166680Writing region info on filesystem at 1731515166680Initializing all the Stores at 1731515166691 (+11 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515166691Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515166691Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515166691Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515166691Cleaning up temporary data from old regions at 1731515166779 (+88 ms)Running coprocessor post-open hooks at 1731515166809 (+30 ms)Region opened successfully at 1731515166810 (+1 ms) 2024-11-13T16:26:06,837 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731515166507 2024-11-13T16:26:06,893 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T16:26:06,893 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T16:26:06,900 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:06,907 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,33401,1731515163358, state=OPEN 2024-11-13T16:26:06,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:06,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:06,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:06,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:06,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:06,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:06,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:06,924 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:06,924 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:06,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T16:26:06,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,33401,1731515163358 in 597 msec 2024-11-13T16:26:06,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T16:26:06,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.3710 sec 2024-11-13T16:26:06,979 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T16:26:06,979 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T16:26:07,006 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:26:07,019 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1] 2024-11-13T16:26:07,050 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:26:07,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41475, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:26:07,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.8710 sec 2024-11-13T16:26:07,106 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731515167106, completionTime=-1 2024-11-13T16:26:07,111 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T16:26:07,112 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T16:26:07,221 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-13T16:26:07,221 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731515227221 2024-11-13T16:26:07,222 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731515287222 2024-11-13T16:26:07,222 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 110 msec 2024-11-13T16:26:07,224 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T16:26:07,263 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,34015,1731515162192-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:07,264 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,34015,1731515162192-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:07,264 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,34015,1731515162192-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:07,267 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f8f85b0bb1d8:34015, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:07,271 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:07,276 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:07,302 DEBUG [master/f8f85b0bb1d8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T16:26:07,336 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.583sec 2024-11-13T16:26:07,340 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T16:26:07,343 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T16:26:07,344 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T16:26:07,345 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T16:26:07,345 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T16:26:07,346 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,34015,1731515162192-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T16:26:07,347 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,34015,1731515162192-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T16:26:07,354 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T16:26:07,355 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T16:26:07,356 INFO [master/f8f85b0bb1d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f8f85b0bb1d8,34015,1731515162192-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:07,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc063d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T16:26:07,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f8f85b0bb1d8,34015,-1 for getting cluster id 2024-11-13T16:26:07,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T16:26:07,443 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6d1291a-e193-41b7-8dec-6caf412160f7' 2024-11-13T16:26:07,448 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T16:26:07,449 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6d1291a-e193-41b7-8dec-6caf412160f7" 2024-11-13T16:26:07,449 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ce37cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T16:26:07,449 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f8f85b0bb1d8,34015,-1] 2024-11-13T16:26:07,456 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T16:26:07,462 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:07,462 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T16:26:07,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93f063a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T16:26:07,474 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:26:07,496 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1] 2024-11-13T16:26:07,496 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:26:07,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:26:07,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:07,538 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:40955/hbase 2024-11-13T16:26:07,571 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=360, OpenFileDescriptor=599, MaxFileDescriptor=1048576, SystemLoadAverage=990, ProcessCount=11, AvailableMemoryMB=3920 2024-11-13T16:26:07,597 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:07,603 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:07,604 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:07,635 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62580229, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-62580229, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:07,657 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62580229/hregion-62580229.1731515167636, exclude list is [], retry=0 2024-11-13T16:26:07,672 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:07,673 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:07,681 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:07,708 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62580229/hregion-62580229.1731515167636 2024-11-13T16:26:07,711 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:07,712 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 610436cbd46767edbc6b3307245062f2, NAME => 'testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:07,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741838_1014 (size=64) 2024-11-13T16:26:07,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741838_1014 (size=64) 2024-11-13T16:26:07,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741838_1014 (size=64) 2024-11-13T16:26:08,218 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:08,224 INFO [StoreOpener-610436cbd46767edbc6b3307245062f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:08,228 INFO [StoreOpener-610436cbd46767edbc6b3307245062f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 610436cbd46767edbc6b3307245062f2 columnFamilyName a 2024-11-13T16:26:08,228 DEBUG [StoreOpener-610436cbd46767edbc6b3307245062f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:08,233 INFO [StoreOpener-610436cbd46767edbc6b3307245062f2-1 {}] regionserver.HStore(327): Store=610436cbd46767edbc6b3307245062f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:08,235 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:08,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:08,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:08,239 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:08,239 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:08,243 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:08,265 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:08,268 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 610436cbd46767edbc6b3307245062f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71878924, jitterRate=0.07107943296432495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:08,271 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 610436cbd46767edbc6b3307245062f2: Writing region info on filesystem at 1731515168218Initializing all the Stores at 1731515168223 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515168223Cleaning up temporary data from old regions at 1731515168239 (+16 ms)Region opened successfully at 1731515168271 (+32 ms) 2024-11-13T16:26:08,272 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 610436cbd46767edbc6b3307245062f2, disabling compactions & flushes 2024-11-13T16:26:08,272 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2. 2024-11-13T16:26:08,272 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2. 2024-11-13T16:26:08,272 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2. after waiting 0 ms 2024-11-13T16:26:08,272 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2. 2024-11-13T16:26:08,278 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2. 2024-11-13T16:26:08,278 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 610436cbd46767edbc6b3307245062f2: Waiting for close lock at 1731515168271Disabling compacts and flushes for region at 1731515168271Disabling writes for close at 1731515168272 (+1 ms)Writing region close event to WAL at 1731515168278 (+6 ms)Closed at 1731515168278 2024-11-13T16:26:08,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741837_1013 (size=93) 2024-11-13T16:26:08,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741837_1013 (size=93) 2024-11-13T16:26:08,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741837_1013 (size=93) 2024-11-13T16:26:08,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:08,317 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62580229:(num 1731515167636) 2024-11-13T16:26:08,321 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T16:26:08,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741839_1015 (size=276) 2024-11-13T16:26:08,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741839_1015 (size=276) 2024-11-13T16:26:08,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741839_1015 (size=276) 2024-11-13T16:26:08,778 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T16:26:08,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741840_1016 (size=230) 2024-11-13T16:26:08,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741840_1016 (size=230) 2024-11-13T16:26:08,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741840_1016 (size=230) 2024-11-13T16:26:08,831 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1, size=276 (276bytes) 2024-11-13T16:26:08,832 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T16:26:08,833 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T16:26:08,833 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1 2024-11-13T16:26:08,844 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1 after 8ms 2024-11-13T16:26:08,858 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:08,860 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1 took 29ms 2024-11-13T16:26:08,868 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1 so closing down 2024-11-13T16:26:08,873 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T16:26:08,876 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T16:26:08,878 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:08,878 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:08,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741841_1017 (size=276) 2024-11-13T16:26:08,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741841_1017 (size=276) 2024-11-13T16:26:08,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741841_1017 (size=276) 2024-11-13T16:26:08,917 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:08,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002 2024-11-13T16:26:08,926 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-13T16:26:08,927 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1, journal: Splitting hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1, size=276 (276bytes) at 1731515168832Finishing writing output for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1 so closing down at 1731515168868 (+36 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000001-wal-1.temp at 1731515168876 (+8 ms)3 split writer threads finished at 1731515168878 (+2 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731515168917 (+39 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002 at 1731515168922 (+5 ms)Processed 2 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1731515168927 (+5 ms) 2024-11-13T16:26:08,948 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2, size=230 (230bytes) 2024-11-13T16:26:08,948 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2 2024-11-13T16:26:08,950 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2 after 2ms 2024-11-13T16:26:08,959 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:08,960 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2 took 12ms 2024-11-13T16:26:08,971 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2 so closing down 2024-11-13T16:26:08,971 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:08,984 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T16:26:08,992 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T16:26:08,993 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:09,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741842_1018 (size=230) 2024-11-13T16:26:09,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741842_1018 (size=230) 2024-11-13T16:26:09,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741842_1018 (size=230) 2024-11-13T16:26:09,021 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:09,030 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:09,033 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-11-13T16:26:09,036 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-13T16:26:09,037 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2, journal: Splitting hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2, size=230 (230bytes) at 1731515168948Finishing writing output for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2 so closing down at 1731515168971 (+23 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002-wal-2.temp at 1731515168992 (+21 ms)3 split writer threads finished at 1731515168993 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731515169021 (+28 ms)Processed 1 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1731515169036 (+15 ms) 2024-11-13T16:26:09,037 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:09,041 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:09,060 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal.1731515169042, exclude list is [], retry=0 2024-11-13T16:26:09,067 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:09,068 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:09,069 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:09,079 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal.1731515169042 2024-11-13T16:26:09,080 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:09,080 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 610436cbd46767edbc6b3307245062f2, NAME => 'testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:09,081 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:09,081 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,081 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,084 INFO [StoreOpener-610436cbd46767edbc6b3307245062f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,086 INFO [StoreOpener-610436cbd46767edbc6b3307245062f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 610436cbd46767edbc6b3307245062f2 columnFamilyName a 2024-11-13T16:26:09,086 DEBUG [StoreOpener-610436cbd46767edbc6b3307245062f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:09,088 INFO [StoreOpener-610436cbd46767edbc6b3307245062f2-1 {}] regionserver.HStore(327): Store=610436cbd46767edbc6b3307245062f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:09,088 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,089 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,094 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,096 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002 2024-11-13T16:26:09,101 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:09,108 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002 2024-11-13T16:26:09,111 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 610436cbd46767edbc6b3307245062f2 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T16:26:09,176 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/.tmp/a/11a5b424166546b9bc25baac9321aad3 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731515168317/Put/seqid=0 2024-11-13T16:26:09,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741844_1020 (size=5170) 2024-11-13T16:26:09,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741844_1020 (size=5170) 2024-11-13T16:26:09,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741844_1020 (size=5170) 2024-11-13T16:26:09,198 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/.tmp/a/11a5b424166546b9bc25baac9321aad3 2024-11-13T16:26:09,247 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/.tmp/a/11a5b424166546b9bc25baac9321aad3 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/a/11a5b424166546b9bc25baac9321aad3 2024-11-13T16:26:09,259 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/a/11a5b424166546b9bc25baac9321aad3, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T16:26:09,265 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 610436cbd46767edbc6b3307245062f2 in 153ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T16:26:09,267 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/0000000000000000002 2024-11-13T16:26:09,268 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,268 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,271 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 610436cbd46767edbc6b3307245062f2 2024-11-13T16:26:09,275 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/610436cbd46767edbc6b3307245062f2/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T16:26:09,276 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 610436cbd46767edbc6b3307245062f2; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72789817, jitterRate=0.08465279638767242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:09,277 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 610436cbd46767edbc6b3307245062f2: Writing region info on filesystem at 1731515169081Initializing all the Stores at 1731515169083 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515169083Obtaining lock to block concurrent updates at 1731515169111 (+28 ms)Preparing flush snapshotting stores in 610436cbd46767edbc6b3307245062f2 at 1731515169111Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731515169114 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731515167598.610436cbd46767edbc6b3307245062f2. at 1731515169114Flushing 610436cbd46767edbc6b3307245062f2/a: creating writer at 1731515169116 (+2 ms)Flushing 610436cbd46767edbc6b3307245062f2/a: appending metadata at 1731515169159 (+43 ms)Flushing 610436cbd46767edbc6b3307245062f2/a: closing flushed file at 1731515169163 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c0a0d29: reopening flushed file at 1731515169244 (+81 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 610436cbd46767edbc6b3307245062f2 in 153ms, sequenceid=2, compaction requested=false; wal=null at 1731515169265 (+21 ms)Cleaning up temporary data from old regions at 1731515169268 (+3 ms)Region opened successfully at 1731515169277 (+9 ms) 2024-11-13T16:26:09,303 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=371 (was 360) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37546 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:56606 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:40955/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:49522 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37468 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:49542 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:56592 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=691 (was 599) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1030 (was 990) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3763 (was 3920) 2024-11-13T16:26:09,316 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=371, OpenFileDescriptor=691, MaxFileDescriptor=1048576, SystemLoadAverage=1030, ProcessCount=11, AvailableMemoryMB=3762 2024-11-13T16:26:09,334 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:09,337 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:09,339 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:09,344 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36326569, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-36326569, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:09,361 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36326569/hregion-36326569.1731515169345, exclude list is [], retry=0 2024-11-13T16:26:09,366 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:09,369 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:09,369 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:09,374 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36326569/hregion-36326569.1731515169345 2024-11-13T16:26:09,374 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:09,375 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1c2c37b6798b326ce33c383b2de9dc12, NAME => 'testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:09,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741846_1022 (size=64) 2024-11-13T16:26:09,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741846_1022 (size=64) 2024-11-13T16:26:09,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741846_1022 (size=64) 2024-11-13T16:26:09,395 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:09,400 INFO [StoreOpener-1c2c37b6798b326ce33c383b2de9dc12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:09,404 INFO [StoreOpener-1c2c37b6798b326ce33c383b2de9dc12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c2c37b6798b326ce33c383b2de9dc12 columnFamilyName a 2024-11-13T16:26:09,404 DEBUG [StoreOpener-1c2c37b6798b326ce33c383b2de9dc12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:09,406 INFO [StoreOpener-1c2c37b6798b326ce33c383b2de9dc12-1 {}] regionserver.HStore(327): Store=1c2c37b6798b326ce33c383b2de9dc12/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:09,406 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:09,408 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:09,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:09,415 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:09,415 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:09,421 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:09,426 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:09,427 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c2c37b6798b326ce33c383b2de9dc12; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75354054, jitterRate=0.12286290526390076}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:09,427 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c2c37b6798b326ce33c383b2de9dc12: Writing region info on filesystem at 1731515169395Initializing all the Stores at 1731515169399 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515169400 (+1 ms)Cleaning up temporary data from old regions at 1731515169415 (+15 ms)Region opened successfully at 1731515169427 (+12 ms) 2024-11-13T16:26:09,428 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1c2c37b6798b326ce33c383b2de9dc12, disabling compactions & flushes 2024-11-13T16:26:09,428 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12. 2024-11-13T16:26:09,428 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12. 2024-11-13T16:26:09,428 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12. after waiting 0 ms 2024-11-13T16:26:09,428 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12. 2024-11-13T16:26:09,436 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12. 2024-11-13T16:26:09,437 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1c2c37b6798b326ce33c383b2de9dc12: Waiting for close lock at 1731515169428Disabling compacts and flushes for region at 1731515169428Disabling writes for close at 1731515169428Writing region close event to WAL at 1731515169436 (+8 ms)Closed at 1731515169436 2024-11-13T16:26:09,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741845_1021 (size=93) 2024-11-13T16:26:09,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741845_1021 (size=93) 2024-11-13T16:26:09,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741845_1021 (size=93) 2024-11-13T16:26:09,464 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:09,464 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36326569:(num 1731515169345) 2024-11-13T16:26:09,468 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T16:26:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741847_1023 (size=276) 2024-11-13T16:26:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741847_1023 (size=276) 2024-11-13T16:26:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741847_1023 (size=276) 2024-11-13T16:26:09,930 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T16:26:09,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741848_1024 (size=230) 2024-11-13T16:26:09,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741848_1024 (size=230) 2024-11-13T16:26:09,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741848_1024 (size=230) 2024-11-13T16:26:10,014 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2, size=230 (230bytes) 2024-11-13T16:26:10,014 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2 2024-11-13T16:26:10,017 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2 after 2ms 2024-11-13T16:26:10,027 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:10,028 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2 took 13ms 2024-11-13T16:26:10,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741831_1007 (size=1321) 2024-11-13T16:26:10,035 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2 so closing down 2024-11-13T16:26:10,035 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:10,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741829_1005 (size=34) 2024-11-13T16:26:10,051 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T16:26:10,055 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T16:26:10,056 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:10,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741828_1004 (size=1189) 2024-11-13T16:26:10,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741832_1008 (size=32) 2024-11-13T16:26:10,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741849_1025 (size=230) 2024-11-13T16:26:10,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741849_1025 (size=230) 2024-11-13T16:26:10,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741849_1025 (size=230) 2024-11-13T16:26:10,122 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:10,133 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002 2024-11-13T16:26:10,133 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 104 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-13T16:26:10,134 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2, journal: Splitting hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2, size=230 (230bytes) at 1731515170014Finishing writing output for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2 so closing down at 1731515170035 (+21 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002-wal-2.temp at 1731515170055 (+20 ms)3 split writer threads finished at 1731515170056 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731515170122 (+66 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002 at 1731515170133 (+11 ms)Processed 1 edits across 1 Regions in 104 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1731515170133 2024-11-13T16:26:10,158 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1, size=276 (276bytes) 2024-11-13T16:26:10,159 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1 2024-11-13T16:26:10,162 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1 after 3ms 2024-11-13T16:26:10,174 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:10,175 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1 took 17ms 2024-11-13T16:26:10,186 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1 so closing down 2024-11-13T16:26:10,187 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:10,198 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T16:26:10,203 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T16:26:10,204 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:10,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741850_1026 (size=276) 2024-11-13T16:26:10,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741850_1026 (size=276) 2024-11-13T16:26:10,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741850_1026 (size=276) 2024-11-13T16:26:10,275 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:10,283 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:10,287 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002, length=230 2024-11-13T16:26:10,290 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002 2024-11-13T16:26:10,292 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-13T16:26:10,293 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1, journal: Splitting hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1, size=276 (276bytes) at 1731515170159Finishing writing output for hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1 so closing down at 1731515170186 (+27 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000001-wal-1.temp at 1731515170203 (+17 ms)3 split writer threads finished at 1731515170204 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731515170275 (+71 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002 at 1731515170290 (+15 ms)Processed 2 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1731515170292 (+2 ms) 2024-11-13T16:26:10,293 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:10,296 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:10,315 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal.1731515170297, exclude list is [], retry=0 2024-11-13T16:26:10,320 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:10,321 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:10,321 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:10,336 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal.1731515170297 2024-11-13T16:26:10,339 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:10,339 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c2c37b6798b326ce33c383b2de9dc12, NAME => 'testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:10,339 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:10,340 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,340 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,350 INFO [StoreOpener-1c2c37b6798b326ce33c383b2de9dc12-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,366 INFO [StoreOpener-1c2c37b6798b326ce33c383b2de9dc12-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c2c37b6798b326ce33c383b2de9dc12 columnFamilyName a 2024-11-13T16:26:10,366 DEBUG [StoreOpener-1c2c37b6798b326ce33c383b2de9dc12-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:10,374 INFO [StoreOpener-1c2c37b6798b326ce33c383b2de9dc12-1 {}] regionserver.HStore(327): Store=1c2c37b6798b326ce33c383b2de9dc12/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:10,375 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,379 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,388 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,391 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002 2024-11-13T16:26:10,404 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:10,415 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002 2024-11-13T16:26:10,415 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1c2c37b6798b326ce33c383b2de9dc12 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T16:26:10,448 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/.tmp/a/8734e0a6aa074284be273f6a000dc5c7 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731515169464/Put/seqid=0 2024-11-13T16:26:10,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741852_1028 (size=5170) 2024-11-13T16:26:10,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741852_1028 (size=5170) 2024-11-13T16:26:10,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741852_1028 (size=5170) 2024-11-13T16:26:10,475 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/.tmp/a/8734e0a6aa074284be273f6a000dc5c7 2024-11-13T16:26:10,504 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/.tmp/a/8734e0a6aa074284be273f6a000dc5c7 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/a/8734e0a6aa074284be273f6a000dc5c7 2024-11-13T16:26:10,522 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/a/8734e0a6aa074284be273f6a000dc5c7, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T16:26:10,522 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1c2c37b6798b326ce33c383b2de9dc12 in 107ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T16:26:10,524 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/0000000000000000002 2024-11-13T16:26:10,525 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,525 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,530 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1c2c37b6798b326ce33c383b2de9dc12 2024-11-13T16:26:10,547 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/1c2c37b6798b326ce33c383b2de9dc12/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T16:26:10,553 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1c2c37b6798b326ce33c383b2de9dc12; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74150585, jitterRate=0.10492981970310211}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:10,553 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1c2c37b6798b326ce33c383b2de9dc12: Writing region info on filesystem at 1731515170340Initializing all the Stores at 1731515170349 (+9 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515170349Obtaining lock to block concurrent updates at 1731515170415 (+66 ms)Preparing flush snapshotting stores in 1c2c37b6798b326ce33c383b2de9dc12 at 1731515170415Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731515170415Flushing stores of testReplayEditsWrittenIntoWAL,,1731515169335.1c2c37b6798b326ce33c383b2de9dc12. at 1731515170415Flushing 1c2c37b6798b326ce33c383b2de9dc12/a: creating writer at 1731515170415Flushing 1c2c37b6798b326ce33c383b2de9dc12/a: appending metadata at 1731515170445 (+30 ms)Flushing 1c2c37b6798b326ce33c383b2de9dc12/a: closing flushed file at 1731515170445Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69702562: reopening flushed file at 1731515170499 (+54 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1c2c37b6798b326ce33c383b2de9dc12 in 107ms, sequenceid=2, compaction requested=false; wal=null at 1731515170522 (+23 ms)Cleaning up temporary data from old regions at 1731515170525 (+3 ms)Region opened successfully at 1731515170553 (+28 ms) 2024-11-13T16:26:10,577 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=385 (was 371) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37620 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@16f166ab[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@19f1e75e[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@17361bc8[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@1331a035[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:56606 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:49656 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:56676 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:49522 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37468 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=773 (was 691) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1030 (was 1030), ProcessCount=11 (was 11), AvailableMemoryMB=3734 (was 3762) 2024-11-13T16:26:10,592 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=773, MaxFileDescriptor=1048576, SystemLoadAverage=1030, ProcessCount=11, AvailableMemoryMB=3733 2024-11-13T16:26:10,617 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:10,621 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:10,622 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:10,627 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-47803791, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-47803791, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:10,648 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-47803791/hregion-47803791.1731515170627, exclude list is [], retry=0 2024-11-13T16:26:10,654 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:10,658 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:10,671 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:10,686 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-47803791/hregion-47803791.1731515170627 2024-11-13T16:26:10,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:10,688 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b91903d356a5851472743fafd5b9e6b6, NAME => 'testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:10,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741854_1030 (size=64) 2024-11-13T16:26:10,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741854_1030 (size=64) 2024-11-13T16:26:10,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741854_1030 (size=64) 2024-11-13T16:26:10,729 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:10,733 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,736 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b91903d356a5851472743fafd5b9e6b6 columnFamilyName a 2024-11-13T16:26:10,737 DEBUG [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:10,738 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(327): Store=b91903d356a5851472743fafd5b9e6b6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:10,738 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,741 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b91903d356a5851472743fafd5b9e6b6 columnFamilyName b 2024-11-13T16:26:10,742 DEBUG [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:10,747 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(327): Store=b91903d356a5851472743fafd5b9e6b6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:10,748 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,751 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b91903d356a5851472743fafd5b9e6b6 columnFamilyName c 2024-11-13T16:26:10,752 DEBUG [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:10,753 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(327): Store=b91903d356a5851472743fafd5b9e6b6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:10,753 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,755 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,757 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,761 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,761 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,765 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:10,771 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:10,780 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:10,781 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b91903d356a5851472743fafd5b9e6b6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67273978, jitterRate=0.002460390329360962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:10,782 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b91903d356a5851472743fafd5b9e6b6: Writing region info on filesystem at 1731515170729Initializing all the Stores at 1731515170733 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515170733Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515170733Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515170733Cleaning up temporary data from old regions at 1731515170761 (+28 ms)Region opened successfully at 1731515170781 (+20 ms) 2024-11-13T16:26:10,782 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b91903d356a5851472743fafd5b9e6b6, disabling compactions & flushes 2024-11-13T16:26:10,782 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:10,782 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:10,782 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. after waiting 0 ms 2024-11-13T16:26:10,782 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:10,783 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:10,783 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b91903d356a5851472743fafd5b9e6b6: Waiting for close lock at 1731515170782Disabling compacts and flushes for region at 1731515170782Disabling writes for close at 1731515170782Writing region close event to WAL at 1731515170783 (+1 ms)Closed at 1731515170783 2024-11-13T16:26:10,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741853_1029 (size=93) 2024-11-13T16:26:10,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741853_1029 (size=93) 2024-11-13T16:26:10,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741853_1029 (size=93) 2024-11-13T16:26:10,820 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:10,820 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-47803791:(num 1731515170627) 2024-11-13T16:26:10,820 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:10,829 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:10,852 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830, exclude list is [], retry=0 2024-11-13T16:26:10,858 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:10,859 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:10,859 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:10,876 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 2024-11-13T16:26:10,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:11,197 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830, size=0 (0bytes) 2024-11-13T16:26:11,197 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 might be still open, length is 0 2024-11-13T16:26:11,197 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 2024-11-13T16:26:11,199 WARN [IPC Server handler 3 on default port 40955 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T16:26:11,200 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 after 3ms 2024-11-13T16:26:12,196 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T16:26:12,330 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T16:26:12,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T16:26:12,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:12,690 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T16:26:12,690 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T16:26:12,691 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T16:26:12,691 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:12,692 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T16:26:12,693 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:13,078 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37664 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33543:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37664 dst: /127.0.0.1:33543 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33543 remote=/127.0.0.1:37664]. Total timeout mills is 60000, 58061 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:13,079 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:56694 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56694 dst: /127.0.0.1:33305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:13,079 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:49680 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36473:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49680 dst: /127.0.0.1:36473 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:13,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741855_1032 (size=200589) 2024-11-13T16:26:13,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741855_1032 (size=200589) 2024-11-13T16:26:15,201 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 after 4004ms 2024-11-13T16:26:15,206 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:15,209 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 took 4012ms 2024-11-13T16:26:15,222 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731515170830.temp 2024-11-13T16:26:15,230 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000000001-wal.1731515170830.temp 2024-11-13T16:26:15,372 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830; continuing. 2024-11-13T16:26:15,372 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 so closing down 2024-11-13T16:26:15,373 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:15,375 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:15,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741856_1033 (size=200597) 2024-11-13T16:26:15,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741856_1033 (size=200597) 2024-11-13T16:26:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741856_1033 (size=200597) 2024-11-13T16:26:15,387 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000000001-wal.1731515170830.temp (wrote 3002 edits, skipped 0 edits in 67 ms) 2024-11-13T16:26:15,390 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000000001-wal.1731515170830.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000003002 2024-11-13T16:26:15,390 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 180 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T16:26:15,390 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830, journal: Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830, size=0 (0bytes) at 1731515171197Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000000001-wal.1731515170830.temp at 1731515175230 (+4033 ms)Split 1024 edits, skipped 0 edits. at 1731515175316 (+86 ms)Split 2048 edits, skipped 0 edits. at 1731515175343 (+27 ms)Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 so closing down at 1731515175373 (+30 ms)3 split writer threads finished at 1731515175375 (+2 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000000001-wal.1731515170830.temp (wrote 3002 edits, skipped 0 edits in 67 ms) at 1731515175387 (+12 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000000001-wal.1731515170830.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000003002 at 1731515175390 (+3 ms)Processed 3002 edits across 1 Regions in 180 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830, size=0, length=0, corrupted=false, cancelled=false at 1731515175390 2024-11-13T16:26:15,392 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515170830 2024-11-13T16:26:15,394 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000003002 2024-11-13T16:26:15,394 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:15,397 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:15,413 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515175398, exclude list is [], retry=0 2024-11-13T16:26:15,417 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:15,418 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:15,418 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:15,429 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515175398 2024-11-13T16:26:15,430 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:15,430 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:15,437 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:15,439 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b91903d356a5851472743fafd5b9e6b6 columnFamilyName a 2024-11-13T16:26:15,439 DEBUG [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:15,440 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(327): Store=b91903d356a5851472743fafd5b9e6b6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:15,441 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:15,443 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b91903d356a5851472743fafd5b9e6b6 columnFamilyName b 2024-11-13T16:26:15,443 DEBUG [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:15,444 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(327): Store=b91903d356a5851472743fafd5b9e6b6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:15,444 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:15,446 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b91903d356a5851472743fafd5b9e6b6 columnFamilyName c 2024-11-13T16:26:15,447 DEBUG [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:15,448 INFO [StoreOpener-b91903d356a5851472743fafd5b9e6b6-1 {}] regionserver.HStore(327): Store=b91903d356a5851472743fafd5b9e6b6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:15,448 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:15,449 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:15,451 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:15,452 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000003002 2024-11-13T16:26:15,460 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:15,519 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T16:26:15,997 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b91903d356a5851472743fafd5b9e6b6 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T16:26:16,043 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/a/f0f5280cd9cf4049930b929721d86c21 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731515170895/Put/seqid=0 2024-11-13T16:26:16,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741858_1035 (size=50463) 2024-11-13T16:26:16,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741858_1035 (size=50463) 2024-11-13T16:26:16,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741858_1035 (size=50463) 2024-11-13T16:26:16,055 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/a/f0f5280cd9cf4049930b929721d86c21 2024-11-13T16:26:16,065 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/a/f0f5280cd9cf4049930b929721d86c21 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/a/f0f5280cd9cf4049930b929721d86c21 2024-11-13T16:26:16,073 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/a/f0f5280cd9cf4049930b929721d86c21, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T16:26:16,074 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for b91903d356a5851472743fafd5b9e6b6 in 76ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T16:26:16,099 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T16:26:16,100 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b91903d356a5851472743fafd5b9e6b6 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T16:26:16,108 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/a/ff9070f0a47d4b85a2ad1ad23f7177cf is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731515170961/Put/seqid=0 2024-11-13T16:26:16,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741859_1036 (size=20072) 2024-11-13T16:26:16,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741859_1036 (size=20072) 2024-11-13T16:26:16,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741859_1036 (size=20072) 2024-11-13T16:26:16,117 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/a/ff9070f0a47d4b85a2ad1ad23f7177cf 2024-11-13T16:26:16,159 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/b/27a5749a496a4984a64301414729e15b is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731515170989/Put/seqid=0 2024-11-13T16:26:16,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741860_1037 (size=35835) 2024-11-13T16:26:16,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741860_1037 (size=35835) 2024-11-13T16:26:16,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741860_1037 (size=35835) 2024-11-13T16:26:16,178 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/b/27a5749a496a4984a64301414729e15b 2024-11-13T16:26:16,187 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/a/ff9070f0a47d4b85a2ad1ad23f7177cf as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/a/ff9070f0a47d4b85a2ad1ad23f7177cf 2024-11-13T16:26:16,195 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/a/ff9070f0a47d4b85a2ad1ad23f7177cf, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T16:26:16,197 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/b/27a5749a496a4984a64301414729e15b as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/b/27a5749a496a4984a64301414729e15b 2024-11-13T16:26:16,205 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/b/27a5749a496a4984a64301414729e15b, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T16:26:16,206 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for b91903d356a5851472743fafd5b9e6b6 in 106ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T16:26:16,220 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T16:26:16,221 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b91903d356a5851472743fafd5b9e6b6 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T16:26:16,230 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/b/6a337e63c5e54bc09b26be15a0bcf238 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731515171008/Put/seqid=0 2024-11-13T16:26:16,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741861_1038 (size=35082) 2024-11-13T16:26:16,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741861_1038 (size=35082) 2024-11-13T16:26:16,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741861_1038 (size=35082) 2024-11-13T16:26:16,239 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/b/6a337e63c5e54bc09b26be15a0bcf238 2024-11-13T16:26:16,263 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/c/ca1fa7fd87c84797af0f4167f317c3da is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731515171053/Put/seqid=0 2024-11-13T16:26:16,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741862_1039 (size=20825) 2024-11-13T16:26:16,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741862_1039 (size=20825) 2024-11-13T16:26:16,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741862_1039 (size=20825) 2024-11-13T16:26:16,277 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/c/ca1fa7fd87c84797af0f4167f317c3da 2024-11-13T16:26:16,285 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/b/6a337e63c5e54bc09b26be15a0bcf238 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/b/6a337e63c5e54bc09b26be15a0bcf238 2024-11-13T16:26:16,293 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/b/6a337e63c5e54bc09b26be15a0bcf238, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T16:26:16,295 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/c/ca1fa7fd87c84797af0f4167f317c3da as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/c/ca1fa7fd87c84797af0f4167f317c3da 2024-11-13T16:26:16,303 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/c/ca1fa7fd87c84797af0f4167f317c3da, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T16:26:16,303 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for b91903d356a5851472743fafd5b9e6b6 in 82ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T16:26:16,315 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731515171128/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:16,321 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000003002 2024-11-13T16:26:16,322 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T16:26:16,322 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b91903d356a5851472743fafd5b9e6b6 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T16:26:16,334 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/c/6bb4cf4b2fb54fc2bdcc8ae52ae619ac is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731515171065/Put/seqid=0 2024-11-13T16:26:16,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741863_1040 (size=50301) 2024-11-13T16:26:16,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741863_1040 (size=50301) 2024-11-13T16:26:16,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741863_1040 (size=50301) 2024-11-13T16:26:16,354 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/c/6bb4cf4b2fb54fc2bdcc8ae52ae619ac 2024-11-13T16:26:16,365 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6bb4cf4b2fb54fc2bdcc8ae52ae619ac 2024-11-13T16:26:16,367 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/.tmp/c/6bb4cf4b2fb54fc2bdcc8ae52ae619ac as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/c/6bb4cf4b2fb54fc2bdcc8ae52ae619ac 2024-11-13T16:26:16,378 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6bb4cf4b2fb54fc2bdcc8ae52ae619ac 2024-11-13T16:26:16,378 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/c/6bb4cf4b2fb54fc2bdcc8ae52ae619ac, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T16:26:16,379 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for b91903d356a5851472743fafd5b9e6b6 in 56ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T16:26:16,380 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/0000000000000003002 2024-11-13T16:26:16,382 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:16,382 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:16,384 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T16:26:16,387 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b91903d356a5851472743fafd5b9e6b6 2024-11-13T16:26:16,391 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenIntoWAL/b91903d356a5851472743fafd5b9e6b6/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T16:26:16,393 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b91903d356a5851472743fafd5b9e6b6; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62292250, jitterRate=-0.07177314162254333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T16:26:16,393 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b91903d356a5851472743fafd5b9e6b6: Writing region info on filesystem at 1731515175430Initializing all the Stores at 1731515175432 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515175432Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515175437 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515175437Cleaning up temporary data from old regions at 1731515176382 (+945 ms)Region opened successfully at 1731515176393 (+11 ms) 2024-11-13T16:26:16,501 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b91903d356a5851472743fafd5b9e6b6, disabling compactions & flushes 2024-11-13T16:26:16,501 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:16,501 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:16,501 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. after waiting 0 ms 2024-11-13T16:26:16,501 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:16,511 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731515170618.b91903d356a5851472743fafd5b9e6b6. 2024-11-13T16:26:16,511 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b91903d356a5851472743fafd5b9e6b6: Waiting for close lock at 1731515176501Disabling compacts and flushes for region at 1731515176501Disabling writes for close at 1731515176501Writing region close event to WAL at 1731515176511 (+10 ms)Closed at 1731515176511 2024-11-13T16:26:16,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741857_1034 (size=93) 2024-11-13T16:26:16,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741857_1034 (size=93) 2024-11-13T16:26:16,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741857_1034 (size=93) 2024-11-13T16:26:16,528 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:16,528 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731515175398) 2024-11-13T16:26:16,540 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=398 (was 385) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390170433_22 at /127.0.0.1:46348 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390170433_22 at /127.0.0.1:33976 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:39039 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:45141 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40955 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390170433_22 at /127.0.0.1:48500 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39039 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-390170433_22 at /127.0.0.1:46332 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:40955 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45141 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=849 (was 773) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1044 (was 1030) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3471 (was 3733) 2024-11-13T16:26:16,553 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=398, OpenFileDescriptor=849, MaxFileDescriptor=1048576, SystemLoadAverage=1044, ProcessCount=11, AvailableMemoryMB=3470 2024-11-13T16:26:16,570 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:16,574 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:16,575 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:16,579 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37814377, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-37814377, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:16,592 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37814377/hregion-37814377.1731515176579, exclude list is [], retry=0 2024-11-13T16:26:16,595 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:16,596 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:16,596 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:16,601 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37814377/hregion-37814377.1731515176579 2024-11-13T16:26:16,601 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:16,602 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 935bcab9c6227b3df1e52990a326e397, NAME => 'test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:16,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741865_1042 (size=43) 2024-11-13T16:26:16,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741865_1042 (size=43) 2024-11-13T16:26:16,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741865_1042 (size=43) 2024-11-13T16:26:16,617 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:16,619 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,621 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 935bcab9c6227b3df1e52990a326e397 columnFamilyName a 2024-11-13T16:26:16,621 DEBUG [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:16,622 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(327): Store=935bcab9c6227b3df1e52990a326e397/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:16,622 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,625 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 935bcab9c6227b3df1e52990a326e397 columnFamilyName b 2024-11-13T16:26:16,625 DEBUG [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:16,626 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(327): Store=935bcab9c6227b3df1e52990a326e397/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:16,626 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,628 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 935bcab9c6227b3df1e52990a326e397 columnFamilyName c 2024-11-13T16:26:16,628 DEBUG [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:16,629 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(327): Store=935bcab9c6227b3df1e52990a326e397/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:16,629 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,630 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,630 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,632 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,633 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,633 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:16,635 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:16,638 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:16,639 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 935bcab9c6227b3df1e52990a326e397; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66977311, jitterRate=-0.0019602924585342407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:16,640 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 935bcab9c6227b3df1e52990a326e397: Writing region info on filesystem at 1731515176618Initializing all the Stores at 1731515176619 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515176619Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515176619Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515176619Cleaning up temporary data from old regions at 1731515176633 (+14 ms)Region opened successfully at 1731515176640 (+7 ms) 2024-11-13T16:26:16,641 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 935bcab9c6227b3df1e52990a326e397, disabling compactions & flushes 2024-11-13T16:26:16,641 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. 2024-11-13T16:26:16,641 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. 2024-11-13T16:26:16,641 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. after waiting 0 ms 2024-11-13T16:26:16,641 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. 2024-11-13T16:26:16,643 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. 2024-11-13T16:26:16,643 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 935bcab9c6227b3df1e52990a326e397: Waiting for close lock at 1731515176641Disabling compacts and flushes for region at 1731515176641Disabling writes for close at 1731515176641Writing region close event to WAL at 1731515176643 (+2 ms)Closed at 1731515176643 2024-11-13T16:26:16,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741864_1041 (size=93) 2024-11-13T16:26:16,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741864_1041 (size=93) 2024-11-13T16:26:16,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:16,668 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37814377:(num 1731515176579) 2024-11-13T16:26:16,669 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:16,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741864_1041 (size=93) 2024-11-13T16:26:16,675 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:16,696 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676, exclude list is [], retry=0 2024-11-13T16:26:16,703 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:16,704 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:16,704 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:16,722 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676 2024-11-13T16:26:16,723 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:16,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741866_1043 (size=200357) 2024-11-13T16:26:17,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741866_1043 (size=200357) 2024-11-13T16:26:17,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741866_1043 (size=200357) 2024-11-13T16:26:17,049 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676, size=195.7 K (200357bytes) 2024-11-13T16:26:17,049 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676 2024-11-13T16:26:17,050 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676 after 1ms 2024-11-13T16:26:17,060 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:17,064 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676 took 15ms 2024-11-13T16:26:17,085 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731515176676.temp 2024-11-13T16:26:17,099 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000000001-wal.1731515176676.temp 2024-11-13T16:26:17,167 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676 so closing down 2024-11-13T16:26:17,167 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:17,172 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:17,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741867_1044 (size=200357) 2024-11-13T16:26:17,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741867_1044 (size=200357) 2024-11-13T16:26:17,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741867_1044 (size=200357) 2024-11-13T16:26:17,182 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000000001-wal.1731515176676.temp (wrote 3000 edits, skipped 0 edits in 18 ms) 2024-11-13T16:26:17,184 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000000001-wal.1731515176676.temp to hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003000 2024-11-13T16:26:17,184 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-11-13T16:26:17,185 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676, journal: Splitting hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676, size=195.7 K (200357bytes) at 1731515177049Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000000001-wal.1731515176676.temp at 1731515177099 (+50 ms)Split 1024 edits, skipped 0 edits. at 1731515177121 (+22 ms)Split 2048 edits, skipped 0 edits. at 1731515177147 (+26 ms)Finishing writing output for hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676 so closing down at 1731515177167 (+20 ms)3 split writer threads finished at 1731515177172 (+5 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000000001-wal.1731515176676.temp (wrote 3000 edits, skipped 0 edits in 18 ms) at 1731515177182 (+10 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000000001-wal.1731515176676.temp to hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003000 at 1731515177184 (+2 ms)Processed 3000 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1731515177184 2024-11-13T16:26:17,187 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515176676 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515176676 2024-11-13T16:26:17,189 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003000 2024-11-13T16:26:17,189 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:17,194 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:17,217 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194, exclude list is [], retry=0 2024-11-13T16:26:17,230 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:17,231 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:17,231 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:17,248 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194 2024-11-13T16:26:17,249 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:17,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741868_1045 (size=200484) 2024-11-13T16:26:17,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741868_1045 (size=200484) 2024-11-13T16:26:17,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741868_1045 (size=200484) 2024-11-13T16:26:17,530 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194, size=195.8 K (200484bytes) 2024-11-13T16:26:17,531 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194 2024-11-13T16:26:17,534 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194 after 3ms 2024-11-13T16:26:17,545 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:17,548 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194 took 18ms 2024-11-13T16:26:17,586 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731515177194.temp 2024-11-13T16:26:17,593 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003001-wal.1731515177194.temp 2024-11-13T16:26:17,612 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194 so closing down 2024-11-13T16:26:17,612 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:17,613 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:17,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741869_1046 (size=200484) 2024-11-13T16:26:17,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741869_1046 (size=200484) 2024-11-13T16:26:17,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741869_1046 (size=200484) 2024-11-13T16:26:17,655 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003001-wal.1731515177194.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-13T16:26:17,657 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003001-wal.1731515177194.temp to hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000006000 2024-11-13T16:26:17,657 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 100 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-11-13T16:26:17,658 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194, journal: Splitting hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194, size=195.8 K (200484bytes) at 1731515177531Split 1024 edits, skipped 0 edits. at 1731515177580 (+49 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003001-wal.1731515177194.temp at 1731515177593 (+13 ms)Split 2048 edits, skipped 0 edits. at 1731515177597 (+4 ms)Finishing writing output for hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194 so closing down at 1731515177612 (+15 ms)3 split writer threads finished at 1731515177613 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003001-wal.1731515177194.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1731515177655 (+42 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003001-wal.1731515177194.temp to hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000006000 at 1731515177657 (+2 ms)Processed 3000 edits across 1 Regions in 100 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1731515177658 (+1 ms) 2024-11-13T16:26:17,661 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177194 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515177194 2024-11-13T16:26:17,665 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000006000 2024-11-13T16:26:17,666 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:17,668 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/test2727-manual,16010,1731515176569, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:17,684 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177669, exclude list is [], retry=0 2024-11-13T16:26:17,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:17,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:17,689 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:17,692 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177669 2024-11-13T16:26:17,693 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:17,693 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 935bcab9c6227b3df1e52990a326e397, NAME => 'test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:17,693 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:17,693 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:17,693 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:17,696 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:17,698 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 935bcab9c6227b3df1e52990a326e397 columnFamilyName a 2024-11-13T16:26:17,698 DEBUG [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:17,699 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(327): Store=935bcab9c6227b3df1e52990a326e397/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:17,699 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:17,700 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 935bcab9c6227b3df1e52990a326e397 columnFamilyName b 2024-11-13T16:26:17,700 DEBUG [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:17,701 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(327): Store=935bcab9c6227b3df1e52990a326e397/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:17,702 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:17,704 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 935bcab9c6227b3df1e52990a326e397 columnFamilyName c 2024-11-13T16:26:17,704 DEBUG [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:17,705 INFO [StoreOpener-935bcab9c6227b3df1e52990a326e397-1 {}] regionserver.HStore(327): Store=935bcab9c6227b3df1e52990a326e397/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:17,705 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:17,706 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:17,713 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:17,714 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003000 2024-11-13T16:26:17,717 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:17,785 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003000 2024-11-13T16:26:17,787 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000006000 2024-11-13T16:26:17,790 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:17,866 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000006000 2024-11-13T16:26:17,867 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 935bcab9c6227b3df1e52990a326e397 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T16:26:17,906 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/a/89a193c32b5643ab8a394b441dd6acf9 is 41, key is test2727/a:100/1731515177256/Put/seqid=0 2024-11-13T16:26:17,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741871_1048 (size=84227) 2024-11-13T16:26:17,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741871_1048 (size=84227) 2024-11-13T16:26:17,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741871_1048 (size=84227) 2024-11-13T16:26:17,927 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/a/89a193c32b5643ab8a394b441dd6acf9 2024-11-13T16:26:17,985 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/b/7fb9a0acf37b4ae38160bcf4758b2265 is 41, key is test2727/b:100/1731515177314/Put/seqid=0 2024-11-13T16:26:18,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741872_1049 (size=84609) 2024-11-13T16:26:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741872_1049 (size=84609) 2024-11-13T16:26:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741872_1049 (size=84609) 2024-11-13T16:26:18,015 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/b/7fb9a0acf37b4ae38160bcf4758b2265 2024-11-13T16:26:18,065 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/c/4a3234ca0b3a4324bb5d51c51924c4ba is 41, key is test2727/c:100/1731515177394/Put/seqid=0 2024-11-13T16:26:18,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741873_1050 (size=84609) 2024-11-13T16:26:18,104 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/c/4a3234ca0b3a4324bb5d51c51924c4ba 2024-11-13T16:26:18,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741873_1050 (size=84609) 2024-11-13T16:26:18,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741873_1050 (size=84609) 2024-11-13T16:26:18,132 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/a/89a193c32b5643ab8a394b441dd6acf9 as hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/a/89a193c32b5643ab8a394b441dd6acf9 2024-11-13T16:26:18,152 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/a/89a193c32b5643ab8a394b441dd6acf9, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T16:26:18,157 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/b/7fb9a0acf37b4ae38160bcf4758b2265 as hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/b/7fb9a0acf37b4ae38160bcf4758b2265 2024-11-13T16:26:18,166 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/b/7fb9a0acf37b4ae38160bcf4758b2265, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T16:26:18,167 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/.tmp/c/4a3234ca0b3a4324bb5d51c51924c4ba as hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/c/4a3234ca0b3a4324bb5d51c51924c4ba 2024-11-13T16:26:18,177 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/c/4a3234ca0b3a4324bb5d51c51924c4ba, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T16:26:18,177 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 935bcab9c6227b3df1e52990a326e397 in 311ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T16:26:18,178 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000003000 2024-11-13T16:26:18,179 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/0000000000000006000 2024-11-13T16:26:18,181 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:18,181 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:18,182 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:18,184 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 935bcab9c6227b3df1e52990a326e397 2024-11-13T16:26:18,187 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/test2727/935bcab9c6227b3df1e52990a326e397/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T16:26:18,188 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 935bcab9c6227b3df1e52990a326e397; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67372816, jitterRate=0.0039331912994384766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:18,190 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 935bcab9c6227b3df1e52990a326e397: Writing region info on filesystem at 1731515177694Initializing all the Stores at 1731515177695 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515177695Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515177696 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515177696Obtaining lock to block concurrent updates at 1731515177867 (+171 ms)Preparing flush snapshotting stores in 935bcab9c6227b3df1e52990a326e397 at 1731515177867Finished memstore snapshotting test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731515177867Flushing stores of test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. at 1731515177867Flushing 935bcab9c6227b3df1e52990a326e397/a: creating writer at 1731515177867Flushing 935bcab9c6227b3df1e52990a326e397/a: appending metadata at 1731515177904 (+37 ms)Flushing 935bcab9c6227b3df1e52990a326e397/a: closing flushed file at 1731515177904Flushing 935bcab9c6227b3df1e52990a326e397/b: creating writer at 1731515177948 (+44 ms)Flushing 935bcab9c6227b3df1e52990a326e397/b: appending metadata at 1731515177982 (+34 ms)Flushing 935bcab9c6227b3df1e52990a326e397/b: closing flushed file at 1731515177982Flushing 935bcab9c6227b3df1e52990a326e397/c: creating writer at 1731515178032 (+50 ms)Flushing 935bcab9c6227b3df1e52990a326e397/c: appending metadata at 1731515178063 (+31 ms)Flushing 935bcab9c6227b3df1e52990a326e397/c: closing flushed file at 1731515178063Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52c93de7: reopening flushed file at 1731515178120 (+57 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6073c67c: reopening flushed file at 1731515178152 (+32 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@182f7a6f: reopening flushed file at 1731515178166 (+14 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 935bcab9c6227b3df1e52990a326e397 in 311ms, sequenceid=6000, compaction requested=false; wal=null at 1731515178177 (+11 ms)Cleaning up temporary data from old regions at 1731515178181 (+4 ms)Region opened successfully at 1731515178190 (+9 ms) 2024-11-13T16:26:18,191 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T16:26:18,192 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 935bcab9c6227b3df1e52990a326e397, disabling compactions & flushes 2024-11-13T16:26:18,192 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. 2024-11-13T16:26:18,192 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. 2024-11-13T16:26:18,192 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. after waiting 0 ms 2024-11-13T16:26:18,192 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. 2024-11-13T16:26:18,193 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731515176571.935bcab9c6227b3df1e52990a326e397. 2024-11-13T16:26:18,193 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 935bcab9c6227b3df1e52990a326e397: Waiting for close lock at 1731515178192Disabling compacts and flushes for region at 1731515178192Disabling writes for close at 1731515178192Writing region close event to WAL at 1731515178193 (+1 ms)Closed at 1731515178193 2024-11-13T16:26:18,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741870_1047 (size=93) 2024-11-13T16:26:18,205 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1731515176569/wal.1731515177669 not finished, retry = 0 2024-11-13T16:26:18,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741870_1047 (size=93) 2024-11-13T16:26:18,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741870_1047 (size=93) 2024-11-13T16:26:18,309 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:18,309 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731515177669) 2024-11-13T16:26:18,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=398 (was 398), OpenFileDescriptor=909 (was 849) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1024 (was 1044), ProcessCount=11 (was 11), AvailableMemoryMB=3223 (was 3470) 2024-11-13T16:26:18,339 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=398, OpenFileDescriptor=909, MaxFileDescriptor=1048576, SystemLoadAverage=1024, ProcessCount=11, AvailableMemoryMB=3222 2024-11-13T16:26:18,360 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:18,371 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:18,372 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731515178372 2024-11-13T16:26:18,399 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372 2024-11-13T16:26:18,415 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:18,434 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cef08d71922c485aa59536e31dedd79b, NAME => 'testSequentialEditLogSeqNum,,1731515178361.cef08d71922c485aa59536e31dedd79b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:18,435 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731515178361.cef08d71922c485aa59536e31dedd79b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:18,435 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,435 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,443 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b doesn't exist for region: cef08d71922c485aa59536e31dedd79b on table testSequentialEditLogSeqNum 2024-11-13T16:26:18,447 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: cef08d71922c485aa59536e31dedd79b on table testSequentialEditLogSeqNum 2024-11-13T16:26:18,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741875_1052 (size=62) 2024-11-13T16:26:18,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741875_1052 (size=62) 2024-11-13T16:26:18,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741875_1052 (size=62) 2024-11-13T16:26:18,487 INFO [StoreOpener-cef08d71922c485aa59536e31dedd79b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,489 INFO [StoreOpener-cef08d71922c485aa59536e31dedd79b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cef08d71922c485aa59536e31dedd79b columnFamilyName a 2024-11-13T16:26:18,489 DEBUG [StoreOpener-cef08d71922c485aa59536e31dedd79b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:18,490 INFO [StoreOpener-cef08d71922c485aa59536e31dedd79b-1 {}] regionserver.HStore(327): Store=cef08d71922c485aa59536e31dedd79b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:18,491 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,493 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,493 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,496 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cef08d71922c485aa59536e31dedd79b 2024-11-13T16:26:18,500 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:18,501 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cef08d71922c485aa59536e31dedd79b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61194426, jitterRate=-0.08813199400901794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:18,503 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cef08d71922c485aa59536e31dedd79b: Writing region info on filesystem at 1731515178435Initializing all the Stores at 1731515178485 (+50 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515178486 (+1 ms)Cleaning up temporary data from old regions at 1731515178493 (+7 ms)Region opened successfully at 1731515178503 (+10 ms) 2024-11-13T16:26:18,549 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cef08d71922c485aa59536e31dedd79b 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T16:26:18,596 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/.tmp/a/0aae0a5ebfef4412b10d14778476d514 is 81, key is testSequentialEditLogSeqNum/a:x0/1731515178503/Put/seqid=0 2024-11-13T16:26:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741876_1053 (size=5833) 2024-11-13T16:26:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741876_1053 (size=5833) 2024-11-13T16:26:18,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741876_1053 (size=5833) 2024-11-13T16:26:18,624 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/.tmp/a/0aae0a5ebfef4412b10d14778476d514 2024-11-13T16:26:18,688 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/.tmp/a/0aae0a5ebfef4412b10d14778476d514 as hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/a/0aae0a5ebfef4412b10d14778476d514 2024-11-13T16:26:18,703 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/a/0aae0a5ebfef4412b10d14778476d514, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T16:26:18,708 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for cef08d71922c485aa59536e31dedd79b in 159ms, sequenceid=13, compaction requested=false 2024-11-13T16:26:18,708 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cef08d71922c485aa59536e31dedd79b: 2024-11-13T16:26:18,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T16:26:18,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T16:26:18,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T16:26:18,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T16:26:18,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T16:26:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741874_1051 (size=1616) 2024-11-13T16:26:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741874_1051 (size=1616) 2024-11-13T16:26:18,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741874_1051 (size=1616) 2024-11-13T16:26:18,768 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372, size=1.6 K (1616bytes) 2024-11-13T16:26:18,768 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372 2024-11-13T16:26:18,770 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372 after 2ms 2024-11-13T16:26:18,776 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:18,776 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372 took 9ms 2024-11-13T16:26:18,780 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372 so closing down 2024-11-13T16:26:18,780 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:18,790 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731515178372.temp 2024-11-13T16:26:18,793 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/0000000000000000003-wal.1731515178372.temp 2024-11-13T16:26:18,794 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:18,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741877_1054 (size=1175) 2024-11-13T16:26:18,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741877_1054 (size=1175) 2024-11-13T16:26:18,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741877_1054 (size=1175) 2024-11-13T16:26:18,820 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/0000000000000000003-wal.1731515178372.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:18,828 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/0000000000000000003-wal.1731515178372.temp to hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/0000000000000000020 2024-11-13T16:26:18,828 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 50 ms; skipped=2; WAL=hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-11-13T16:26:18,829 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372, journal: Splitting hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372, size=1.6 K (1616bytes) at 1731515178768Finishing writing output for hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372 so closing down at 1731515178780 (+12 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/0000000000000000003-wal.1731515178372.temp at 1731515178793 (+13 ms)3 split writer threads finished at 1731515178794 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/0000000000000000003-wal.1731515178372.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731515178820 (+26 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/0000000000000000003-wal.1731515178372.temp to hdfs://localhost:40955/hbase/data/default/testSequentialEditLogSeqNum/cef08d71922c485aa59536e31dedd79b/recovered.edits/0000000000000000020 at 1731515178828 (+8 ms)Processed 17 edits across 1 Regions in 50 ms; skipped=2; WAL=hdfs://localhost:40955/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731515178359/wal.1731515178372, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1731515178828 2024-11-13T16:26:18,850 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=403 (was 398) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:34090 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:48500 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:46470 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=945 (was 909) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1024 (was 1024), ProcessCount=11 (was 11), AvailableMemoryMB=3114 (was 3222) 2024-11-13T16:26:18,867 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=403, OpenFileDescriptor=946, MaxFileDescriptor=1048576, SystemLoadAverage=1024, ProcessCount=11, AvailableMemoryMB=3112 2024-11-13T16:26:18,890 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:18,894 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:18,896 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:18,900 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-20388006, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-20388006, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:18,919 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-20388006/hregion-20388006.1731515178901, exclude list is [], retry=0 2024-11-13T16:26:18,923 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:18,923 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:18,923 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:18,971 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-20388006/hregion-20388006.1731515178901 2024-11-13T16:26:18,975 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:18,975 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 27b571126a07e8619e0c7ccb54aee5ce, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:19,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741879_1056 (size=70) 2024-11-13T16:26:19,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741879_1056 (size=70) 2024-11-13T16:26:19,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741879_1056 (size=70) 2024-11-13T16:26:19,027 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:19,043 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,053 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName a 2024-11-13T16:26:19,053 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:19,056 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:19,056 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,064 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName b 2024-11-13T16:26:19,064 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:19,066 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:19,066 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,079 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName c 2024-11-13T16:26:19,079 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:19,082 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:19,082 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,087 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,087 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,102 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,102 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,103 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:19,115 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,131 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:19,132 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 27b571126a07e8619e0c7ccb54aee5ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72455876, jitterRate=0.07967668771743774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:19,133 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 27b571126a07e8619e0c7ccb54aee5ce: Writing region info on filesystem at 1731515179027Initializing all the Stores at 1731515179035 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515179035Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515179038 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515179039 (+1 ms)Cleaning up temporary data from old regions at 1731515179102 (+63 ms)Region opened successfully at 1731515179133 (+31 ms) 2024-11-13T16:26:19,134 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 27b571126a07e8619e0c7ccb54aee5ce, disabling compactions & flushes 2024-11-13T16:26:19,134 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. 2024-11-13T16:26:19,134 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. 2024-11-13T16:26:19,134 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. after waiting 0 ms 2024-11-13T16:26:19,134 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. 2024-11-13T16:26:19,135 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. 2024-11-13T16:26:19,135 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 27b571126a07e8619e0c7ccb54aee5ce: Waiting for close lock at 1731515179134Disabling compacts and flushes for region at 1731515179134Disabling writes for close at 1731515179134Writing region close event to WAL at 1731515179135 (+1 ms)Closed at 1731515179135 2024-11-13T16:26:19,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741878_1055 (size=93) 2024-11-13T16:26:19,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741878_1055 (size=93) 2024-11-13T16:26:19,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741878_1055 (size=93) 2024-11-13T16:26:19,146 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:19,146 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-20388006:(num 1731515178901) 2024-11-13T16:26:19,146 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:19,152 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:19,171 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152, exclude list is [], retry=0 2024-11-13T16:26:19,177 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:19,178 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:19,178 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:19,191 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 2024-11-13T16:26:19,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:19,195 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 27b571126a07e8619e0c7ccb54aee5ce, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:19,195 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:19,196 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,196 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,198 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,200 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName a 2024-11-13T16:26:19,200 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:19,201 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:19,202 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,203 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName b 2024-11-13T16:26:19,203 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:19,204 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:19,204 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,206 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName c 2024-11-13T16:26:19,206 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:19,207 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:19,208 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,209 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,214 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,214 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,215 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:19,218 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:19,219 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 27b571126a07e8619e0c7ccb54aee5ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67456812, jitterRate=0.005184829235076904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:19,220 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 27b571126a07e8619e0c7ccb54aee5ce: Writing region info on filesystem at 1731515179196Initializing all the Stores at 1731515179198 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515179198Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515179198Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515179198Cleaning up temporary data from old regions at 1731515179214 (+16 ms)Region opened successfully at 1731515179220 (+6 ms) 2024-11-13T16:26:19,228 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731515179226/Put/seqid=0 2024-11-13T16:26:19,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741881_1058 (size=4826) 2024-11-13T16:26:19,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741881_1058 (size=4826) 2024-11-13T16:26:19,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741881_1058 (size=4826) 2024-11-13T16:26:19,295 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40955/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 27b571126a07e8619e0c7ccb54aee5ce/a 2024-11-13T16:26:19,310 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T16:26:19,311 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T16:26:19,311 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 27b571126a07e8619e0c7ccb54aee5ce: 2024-11-13T16:26:19,319 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/a/9ae66e87ef5147aaa73b16e0a3c14c71_SeqId_3_ 2024-11-13T16:26:19,320 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40955/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 27b571126a07e8619e0c7ccb54aee5ce/a as hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/a/9ae66e87ef5147aaa73b16e0a3c14c71_SeqId_3_ - updating store file list. 2024-11-13T16:26:19,329 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9ae66e87ef5147aaa73b16e0a3c14c71_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T16:26:19,330 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/a/9ae66e87ef5147aaa73b16e0a3c14c71_SeqId_3_ into 27b571126a07e8619e0c7ccb54aee5ce/a 2024-11-13T16:26:19,330 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40955/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 27b571126a07e8619e0c7ccb54aee5ce/a (new location: hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/a/9ae66e87ef5147aaa73b16e0a3c14c71_SeqId_3_) 2024-11-13T16:26:19,405 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152, size=0 (0bytes) 2024-11-13T16:26:19,405 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 might be still open, length is 0 2024-11-13T16:26:19,405 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 2024-11-13T16:26:19,405 WARN [IPC Server handler 2 on default port 40955 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T16:26:19,406 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 after 1ms 2024-11-13T16:26:22,064 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:34180 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36473:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34180 dst: /127.0.0.1:36473 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36473 remote=/127.0.0.1:34180]. Total timeout mills is 60000, 57276 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:22,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:48674 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48674 dst: /127.0.0.1:33305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:22,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:46572 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33543:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46572 dst: /127.0.0.1:33543 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:22,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741880_1059 (size=434) 2024-11-13T16:26:22,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741880_1059 (size=434) 2024-11-13T16:26:22,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T16:26:22,686 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:22,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T16:26:22,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:23,407 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 after 4001ms 2024-11-13T16:26:23,410 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:23,410 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 took 4006ms 2024-11-13T16:26:23,413 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152; continuing. 2024-11-13T16:26:23,413 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 so closing down 2024-11-13T16:26:23,413 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:23,416 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731515179152.temp 2024-11-13T16:26:23,417 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005-wal.1731515179152.temp 2024-11-13T16:26:23,417 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:23,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741882_1060 (size=236) 2024-11-13T16:26:23,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741882_1060 (size=236) 2024-11-13T16:26:23,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741882_1060 (size=236) 2024-11-13T16:26:23,433 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005-wal.1731515179152.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:23,435 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005-wal.1731515179152.temp to hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005 2024-11-13T16:26:23,436 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=1; WAL=hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T16:26:23,436 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152, journal: Splitting hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152, size=0 (0bytes) at 1731515179405Finishing writing output for hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 so closing down at 1731515183413 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005-wal.1731515179152.temp at 1731515183417 (+4 ms)3 split writer threads finished at 1731515183417Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005-wal.1731515179152.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731515183433 (+16 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005-wal.1731515179152.temp to hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005 at 1731515183435 (+2 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=1; WAL=hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152, size=0, length=0, corrupted=false, cancelled=false at 1731515183436 (+1 ms) 2024-11-13T16:26:23,439 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515179152 2024-11-13T16:26:23,440 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005 2024-11-13T16:26:23,440 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:23,442 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:23,455 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515183442, exclude list is [], retry=0 2024-11-13T16:26:23,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:23,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:23,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:23,460 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515183442 2024-11-13T16:26:23,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:23,461 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 27b571126a07e8619e0c7ccb54aee5ce, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:23,461 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:23,461 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,461 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,463 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,464 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName a 2024-11-13T16:26:23,464 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:23,471 DEBUG [StoreFileOpener-27b571126a07e8619e0c7ccb54aee5ce-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9ae66e87ef5147aaa73b16e0a3c14c71_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T16:26:23,471 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/a/9ae66e87ef5147aaa73b16e0a3c14c71_SeqId_3_ 2024-11-13T16:26:23,471 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:23,472 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,473 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName b 2024-11-13T16:26:23,473 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:23,474 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:23,474 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,475 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27b571126a07e8619e0c7ccb54aee5ce columnFamilyName c 2024-11-13T16:26:23,475 DEBUG [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:23,476 INFO [StoreOpener-27b571126a07e8619e0c7ccb54aee5ce-1 {}] regionserver.HStore(327): Store=27b571126a07e8619e0c7ccb54aee5ce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:23,476 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,477 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,480 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005 2024-11-13T16:26:23,482 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:23,484 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005 2024-11-13T16:26:23,484 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 27b571126a07e8619e0c7ccb54aee5ce 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T16:26:23,501 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/.tmp/a/c2ead9e5b1424a62830b2da45ef69ed8 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731515179339/Put/seqid=0 2024-11-13T16:26:23,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741884_1062 (size=5149) 2024-11-13T16:26:23,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741884_1062 (size=5149) 2024-11-13T16:26:23,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741884_1062 (size=5149) 2024-11-13T16:26:23,508 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/.tmp/a/c2ead9e5b1424a62830b2da45ef69ed8 2024-11-13T16:26:23,517 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/.tmp/a/c2ead9e5b1424a62830b2da45ef69ed8 as hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/a/c2ead9e5b1424a62830b2da45ef69ed8 2024-11-13T16:26:23,523 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/a/c2ead9e5b1424a62830b2da45ef69ed8, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T16:26:23,524 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 27b571126a07e8619e0c7ccb54aee5ce in 40ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T16:26:23,525 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/0000000000000000005 2024-11-13T16:26:23,526 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,526 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,527 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:23,529 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 27b571126a07e8619e0c7ccb54aee5ce 2024-11-13T16:26:23,531 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/27b571126a07e8619e0c7ccb54aee5ce/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T16:26:23,532 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 27b571126a07e8619e0c7ccb54aee5ce; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70162774, jitterRate=0.045506805181503296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:23,532 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 27b571126a07e8619e0c7ccb54aee5ce: Writing region info on filesystem at 1731515183462Initializing all the Stores at 1731515183463 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515183463Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515183463Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515183463Obtaining lock to block concurrent updates at 1731515183484 (+21 ms)Preparing flush snapshotting stores in 27b571126a07e8619e0c7ccb54aee5ce at 1731515183484Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731515183485 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. at 1731515183485Flushing 27b571126a07e8619e0c7ccb54aee5ce/a: creating writer at 1731515183485Flushing 27b571126a07e8619e0c7ccb54aee5ce/a: appending metadata at 1731515183500 (+15 ms)Flushing 27b571126a07e8619e0c7ccb54aee5ce/a: closing flushed file at 1731515183500Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4919a49d: reopening flushed file at 1731515183515 (+15 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 27b571126a07e8619e0c7ccb54aee5ce in 40ms, sequenceid=5, compaction requested=false; wal=null at 1731515183524 (+9 ms)Cleaning up temporary data from old regions at 1731515183526 (+2 ms)Region opened successfully at 1731515183532 (+6 ms) 2024-11-13T16:26:23,537 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 27b571126a07e8619e0c7ccb54aee5ce, disabling compactions & flushes 2024-11-13T16:26:23,537 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. 2024-11-13T16:26:23,537 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. 2024-11-13T16:26:23,537 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. after waiting 0 ms 2024-11-13T16:26:23,537 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. 2024-11-13T16:26:23,538 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731515178891.27b571126a07e8619e0c7ccb54aee5ce. 2024-11-13T16:26:23,538 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 27b571126a07e8619e0c7ccb54aee5ce: Waiting for close lock at 1731515183537Disabling compacts and flushes for region at 1731515183537Disabling writes for close at 1731515183537Writing region close event to WAL at 1731515183538 (+1 ms)Closed at 1731515183538 2024-11-13T16:26:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741883_1061 (size=93) 2024-11-13T16:26:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741883_1061 (size=93) 2024-11-13T16:26:23,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741883_1061 (size=93) 2024-11-13T16:26:23,546 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:23,546 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731515183442) 2024-11-13T16:26:23,563 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=409 (was 403) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:40955 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1248342349_22 at /127.0.0.1:34222 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1248342349_22 at /127.0.0.1:48678 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40955 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1010 (was 946) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=958 (was 1024), ProcessCount=11 (was 11), AvailableMemoryMB=2943 (was 3112) 2024-11-13T16:26:23,580 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=409, OpenFileDescriptor=1010, MaxFileDescriptor=1048576, SystemLoadAverage=958, ProcessCount=11, AvailableMemoryMB=2941 2024-11-13T16:26:23,600 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:23,607 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T16:26:23,613 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:23,615 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@705781a8 2024-11-13T16:26:23,617 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T16:26:23,619 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T16:26:23,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T16:26:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T16:26:23,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T16:26:23,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T16:26:23,643 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:23,646 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T16:26:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:26:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741885_1063 (size=694) 2024-11-13T16:26:23,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741885_1063 (size=694) 2024-11-13T16:26:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741885_1063 (size=694) 2024-11-13T16:26:23,664 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 32b359af0da6a75b36c596a655fd4546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19 2024-11-13T16:26:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741886_1064 (size=77) 2024-11-13T16:26:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741886_1064 (size=77) 2024-11-13T16:26:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741886_1064 (size=77) 2024-11-13T16:26:23,683 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:23,684 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 32b359af0da6a75b36c596a655fd4546, disabling compactions & flushes 2024-11-13T16:26:23,684 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:23,684 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:23,684 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. after waiting 0 ms 2024-11-13T16:26:23,684 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:23,684 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:23,684 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 32b359af0da6a75b36c596a655fd4546: Waiting for close lock at 1731515183684Disabling compacts and flushes for region at 1731515183684Disabling writes for close at 1731515183684Writing region close event to WAL at 1731515183684Closed at 1731515183684 2024-11-13T16:26:23,686 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T16:26:23,693 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731515183687"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731515183687"}]},"ts":"1731515183687"} 2024-11-13T16:26:23,700 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T16:26:23,702 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T16:26:23,705 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731515183702"}]},"ts":"1731515183702"} 2024-11-13T16:26:23,710 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T16:26:23,711 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f8f85b0bb1d8=0} racks are {/default-rack=0} 2024-11-13T16:26:23,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T16:26:23,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T16:26:23,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T16:26:23,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T16:26:23,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T16:26:23,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T16:26:23,713 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T16:26:23,713 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T16:26:23,713 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T16:26:23,713 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T16:26:23,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN}] 2024-11-13T16:26:23,719 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN 2024-11-13T16:26:23,722 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN; state=OFFLINE, location=f8f85b0bb1d8,33401,1731515163358; forceNewPlan=false, retain=false 2024-11-13T16:26:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:26:23,877 INFO [f8f85b0bb1d8:34015 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T16:26:23,878 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=OPENING, regionLocation=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:23,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN because future has completed 2024-11-13T16:26:23,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358}] 2024-11-13T16:26:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:26:24,063 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,063 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 32b359af0da6a75b36c596a655fd4546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:24,064 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,064 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:24,065 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,065 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,071 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,073 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf1 2024-11-13T16:26:24,073 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:24,074 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:24,074 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,079 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf2 2024-11-13T16:26:24,079 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:24,083 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:24,083 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,084 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,085 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,087 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,087 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,089 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T16:26:24,095 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,098 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:24,099 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 32b359af0da6a75b36c596a655fd4546; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61918008, jitterRate=-0.07734978199005127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T16:26:24,100 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,101 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 32b359af0da6a75b36c596a655fd4546: Running coprocessor pre-open hook at 1731515184065Writing region info on filesystem at 1731515184065Initializing all the Stores at 1731515184066 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515184066Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515184070 (+4 ms)Cleaning up temporary data from old regions at 1731515184087 (+17 ms)Running coprocessor post-open hooks at 1731515184100 (+13 ms)Region opened successfully at 1731515184101 (+1 ms) 2024-11-13T16:26:24,103 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., pid=6, masterSystemTime=1731515184044 2024-11-13T16:26:24,108 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,108 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,110 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=OPEN, openSeqNum=2, regionLocation=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:24,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358 because future has completed 2024-11-13T16:26:24,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T16:26:24,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358 in 231 msec 2024-11-13T16:26:24,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T16:26:24,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN in 406 msec 2024-11-13T16:26:24,127 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T16:26:24,128 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731515184127"}]},"ts":"1731515184127"} 2024-11-13T16:26:24,130 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T16:26:24,132 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T16:26:24,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 506 msec 2024-11-13T16:26:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T16:26:24,275 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T16:26:24,275 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T16:26:24,276 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T16:26:24,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T16:26:24,284 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T16:26:24,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T16:26:24,302 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=2] 2024-11-13T16:26:24,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=32b359af0da6a75b36c596a655fd4546, source=f8f85b0bb1d8,33401,1731515163358, destination=f8f85b0bb1d8,37847,1731515163204, warming up region on f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T16:26:24,333 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T16:26:24,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=32b359af0da6a75b36c596a655fd4546, source=f8f85b0bb1d8,33401,1731515163358, destination=f8f85b0bb1d8,37847,1731515163204, running balancer 2024-11-13T16:26:24,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:26:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE 2024-11-13T16:26:24,335 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE 2024-11-13T16:26:24,338 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=CLOSING, regionLocation=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:24,340 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(7855): Warmup {ENCODED => 32b359af0da6a75b36c596a655fd4546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:24,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE because future has completed 2024-11-13T16:26:24,347 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34015 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=f8f85b0bb1d8,33401,1731515163358, table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T16:26:24,351 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,356 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf1 2024-11-13T16:26:24,356 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:24,357 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:24,357 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,359 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf2 2024-11-13T16:26:24,359 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:24,362 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(1722): Closing 32b359af0da6a75b36c596a655fd4546, disabling compactions & flushes 2024-11-13T16:26:24,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. after waiting 0 ms 2024-11-13T16:26:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,368 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T16:26:24,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358}] 2024-11-13T16:26:24,370 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] regionserver.HRegion(1676): Region close journal for 32b359af0da6a75b36c596a655fd4546: Waiting for close lock at 1731515184363Disabling compacts and flushes for region at 1731515184363Disabling writes for close at 1731515184363Writing region close event to WAL at 1731515184370 (+7 ms)Closed at 1731515184370 2024-11-13T16:26:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T16:26:24,532 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:24,533 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T16:26:24,534 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 32b359af0da6a75b36c596a655fd4546, disabling compactions & flushes 2024-11-13T16:26:24,534 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,534 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,534 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. after waiting 0 ms 2024-11-13T16:26:24,534 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:24,534 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 32b359af0da6a75b36c596a655fd4546 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T16:26:24,559 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf1/13ad8f0893ab48679eda3bbd561c1336 is 35, key is r1/cf1:q/1731515184305/Put/seqid=0 2024-11-13T16:26:24,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741887_1065 (size=4783) 2024-11-13T16:26:24,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741887_1065 (size=4783) 2024-11-13T16:26:24,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741887_1065 (size=4783) 2024-11-13T16:26:24,980 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf1/13ad8f0893ab48679eda3bbd561c1336 2024-11-13T16:26:24,991 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf1/13ad8f0893ab48679eda3bbd561c1336 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/13ad8f0893ab48679eda3bbd561c1336 2024-11-13T16:26:25,000 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/13ad8f0893ab48679eda3bbd561c1336, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T16:26:25,002 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 32b359af0da6a75b36c596a655fd4546 in 468ms, sequenceid=5, compaction requested=false 2024-11-13T16:26:25,008 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:26:25,011 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:25,012 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 32b359af0da6a75b36c596a655fd4546: Waiting for close lock at 1731515184533Running coprocessor pre-close hooks at 1731515184533Disabling compacts and flushes for region at 1731515184533Disabling writes for close at 1731515184534 (+1 ms)Obtaining lock to block concurrent updates at 1731515184534Preparing flush snapshotting stores in 32b359af0da6a75b36c596a655fd4546 at 1731515184534Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731515184535 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. at 1731515184536 (+1 ms)Flushing 32b359af0da6a75b36c596a655fd4546/cf1: creating writer at 1731515184536Flushing 32b359af0da6a75b36c596a655fd4546/cf1: appending metadata at 1731515184559 (+23 ms)Flushing 32b359af0da6a75b36c596a655fd4546/cf1: closing flushed file at 1731515184559Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73217b8d: reopening flushed file at 1731515184989 (+430 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 32b359af0da6a75b36c596a655fd4546 in 468ms, sequenceid=5, compaction requested=false at 1731515185002 (+13 ms)Writing region close event to WAL at 1731515185003 (+1 ms)Running coprocessor post-close hooks at 1731515185009 (+6 ms)Closed at 1731515185011 (+2 ms) 2024-11-13T16:26:25,012 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 32b359af0da6a75b36c596a655fd4546 move to f8f85b0bb1d8,37847,1731515163204 record at close sequenceid=5 2024-11-13T16:26:25,016 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,017 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=CLOSED 2024-11-13T16:26:25,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358 because future has completed 2024-11-13T16:26:25,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T16:26:25,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358 in 661 msec 2024-11-13T16:26:25,047 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE; state=CLOSED, location=f8f85b0bb1d8,37847,1731515163204; forceNewPlan=false, retain=false 2024-11-13T16:26:25,198 INFO [f8f85b0bb1d8:34015 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T16:26:25,199 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=OPENING, regionLocation=f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:25,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE because future has completed 2024-11-13T16:26:25,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204}] 2024-11-13T16:26:25,371 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:25,371 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 32b359af0da6a75b36c596a655fd4546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:25,371 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,372 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:25,372 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,372 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,379 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,381 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf1 2024-11-13T16:26:25,381 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:25,400 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/13ad8f0893ab48679eda3bbd561c1336 2024-11-13T16:26:25,401 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:25,401 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,403 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf2 2024-11-13T16:26:25,403 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:25,404 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:25,405 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,407 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,412 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,415 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,415 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,417 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T16:26:25,420 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,421 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 32b359af0da6a75b36c596a655fd4546; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72116911, jitterRate=0.07462571561336517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T16:26:25,422 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:25,423 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 32b359af0da6a75b36c596a655fd4546: Running coprocessor pre-open hook at 1731515185372Writing region info on filesystem at 1731515185372Initializing all the Stores at 1731515185374 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515185374Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515185379 (+5 ms)Cleaning up temporary data from old regions at 1731515185415 (+36 ms)Running coprocessor post-open hooks at 1731515185422 (+7 ms)Region opened successfully at 1731515185423 (+1 ms) 2024-11-13T16:26:25,425 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., pid=9, masterSystemTime=1731515185358 2024-11-13T16:26:25,429 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:25,429 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:25,432 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=OPEN, openSeqNum=9, regionLocation=f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:25,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204 because future has completed 2024-11-13T16:26:25,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T16:26:25,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204 in 236 msec 2024-11-13T16:26:25,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE in 1.1100 sec 2024-11-13T16:26:25,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T16:26:25,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:26:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:44782 deadline: 1731515245502, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f8f85b0bb1d8 port=37847 startCode=1731515163204. As of locationSeqNum=5. 2024-11-13T16:26:25,512 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f8f85b0bb1d8 port=37847 startCode=1731515163204. As of locationSeqNum=5. 2024-11-13T16:26:25,513 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f8f85b0bb1d8 port=37847 startCode=1731515163204. As of locationSeqNum=5. 2024-11-13T16:26:25,513 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,37847,1731515163204, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f8f85b0bb1d8 port=37847 startCode=1731515163204. As of locationSeqNum=5. 2024-11-13T16:26:25,623 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:26:25,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:26:25,645 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 32b359af0da6a75b36c596a655fd4546 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T16:26:25,673 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf1/656e67665f164aa998e51382836c6df7 is 29, key is r1/cf1:/1731515185628/DeleteFamily/seqid=0 2024-11-13T16:26:25,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741888_1066 (size=4906) 2024-11-13T16:26:25,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741888_1066 (size=4906) 2024-11-13T16:26:25,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741888_1066 (size=4906) 2024-11-13T16:26:25,721 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf1/656e67665f164aa998e51382836c6df7 2024-11-13T16:26:25,738 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 656e67665f164aa998e51382836c6df7 2024-11-13T16:26:25,756 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf2/a83ddb58162142efa1b15ac33111fcea is 29, key is r1/cf2:/1731515185628/DeleteFamily/seqid=0 2024-11-13T16:26:25,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741889_1067 (size=4906) 2024-11-13T16:26:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741889_1067 (size=4906) 2024-11-13T16:26:25,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741889_1067 (size=4906) 2024-11-13T16:26:25,795 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf2/a83ddb58162142efa1b15ac33111fcea 2024-11-13T16:26:25,806 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a83ddb58162142efa1b15ac33111fcea 2024-11-13T16:26:25,809 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf1/656e67665f164aa998e51382836c6df7 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/656e67665f164aa998e51382836c6df7 2024-11-13T16:26:25,828 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 656e67665f164aa998e51382836c6df7 2024-11-13T16:26:25,829 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/656e67665f164aa998e51382836c6df7, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T16:26:25,831 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf2/a83ddb58162142efa1b15ac33111fcea as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/a83ddb58162142efa1b15ac33111fcea 2024-11-13T16:26:25,840 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a83ddb58162142efa1b15ac33111fcea 2024-11-13T16:26:25,840 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/a83ddb58162142efa1b15ac33111fcea, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T16:26:25,841 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 32b359af0da6a75b36c596a655fd4546 in 196ms, sequenceid=12, compaction requested=false 2024-11-13T16:26:25,842 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 32b359af0da6a75b36c596a655fd4546: 2024-11-13T16:26:25,846 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T16:26:25,847 DEBUG [Time-limited test {}] regionserver.HStore(1541): 32b359af0da6a75b36c596a655fd4546/cf1 is initiating major compaction (all files) 2024-11-13T16:26:25,848 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T16:26:25,848 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:25,848 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 32b359af0da6a75b36c596a655fd4546/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:25,849 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/13ad8f0893ab48679eda3bbd561c1336, hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/656e67665f164aa998e51382836c6df7] into tmpdir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp, totalSize=9.5 K 2024-11-13T16:26:25,850 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 13ad8f0893ab48679eda3bbd561c1336, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731515184305 2024-11-13T16:26:25,851 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 656e67665f164aa998e51382836c6df7, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T16:26:25,876 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 32b359af0da6a75b36c596a655fd4546#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T16:26:25,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741890_1068 (size=4626) 2024-11-13T16:26:25,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741890_1068 (size=4626) 2024-11-13T16:26:25,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741890_1068 (size=4626) 2024-11-13T16:26:25,955 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf1/63c27a0f95ed483c8dc979cee723ad87 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/63c27a0f95ed483c8dc979cee723ad87 2024-11-13T16:26:25,988 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 32b359af0da6a75b36c596a655fd4546/cf1 of 32b359af0da6a75b36c596a655fd4546 into 63c27a0f95ed483c8dc979cee723ad87(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T16:26:25,988 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 32b359af0da6a75b36c596a655fd4546: 2024-11-13T16:26:25,989 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T16:26:25,989 DEBUG [Time-limited test {}] regionserver.HStore(1541): 32b359af0da6a75b36c596a655fd4546/cf2 is initiating major compaction (all files) 2024-11-13T16:26:25,989 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T16:26:25,989 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T16:26:25,990 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 32b359af0da6a75b36c596a655fd4546/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:25,990 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/a83ddb58162142efa1b15ac33111fcea] into tmpdir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp, totalSize=4.8 K 2024-11-13T16:26:25,991 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a83ddb58162142efa1b15ac33111fcea, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T16:26:26,008 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 32b359af0da6a75b36c596a655fd4546#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T16:26:26,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741891_1069 (size=4592) 2024-11-13T16:26:26,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741891_1069 (size=4592) 2024-11-13T16:26:26,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741891_1069 (size=4592) 2024-11-13T16:26:26,057 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/.tmp/cf2/42f69f18b99e49b9916b06a0852a9611 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/42f69f18b99e49b9916b06a0852a9611 2024-11-13T16:26:26,069 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 32b359af0da6a75b36c596a655fd4546/cf2 of 32b359af0da6a75b36c596a655fd4546 into 42f69f18b99e49b9916b06a0852a9611(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T16:26:26,069 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 32b359af0da6a75b36c596a655fd4546: 2024-11-13T16:26:26,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=32b359af0da6a75b36c596a655fd4546, source=f8f85b0bb1d8,37847,1731515163204, destination=f8f85b0bb1d8,33401,1731515163358, warming up region on f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:26,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=32b359af0da6a75b36c596a655fd4546, source=f8f85b0bb1d8,37847,1731515163204, destination=f8f85b0bb1d8,33401,1731515163358, running balancer 2024-11-13T16:26:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE 2024-11-13T16:26:26,078 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE 2024-11-13T16:26:26,080 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=CLOSING, regionLocation=f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:26,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(7855): Warmup {ENCODED => 32b359af0da6a75b36c596a655fd4546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:26,085 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,087 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf1 2024-11-13T16:26:26,087 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:26,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE because future has completed 2024-11-13T16:26:26,090 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T16:26:26,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204}] 2024-11-13T16:26:26,127 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/13ad8f0893ab48679eda3bbd561c1336 2024-11-13T16:26:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T16:26:26,140 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/63c27a0f95ed483c8dc979cee723ad87 2024-11-13T16:26:26,155 INFO [StoreFileOpener-32b359af0da6a75b36c596a655fd4546-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 656e67665f164aa998e51382836c6df7 2024-11-13T16:26:26,158 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/656e67665f164aa998e51382836c6df7 2024-11-13T16:26:26,159 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:26,159 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,161 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf2 2024-11-13T16:26:26,162 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:26,173 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/42f69f18b99e49b9916b06a0852a9611 2024-11-13T16:26:26,182 INFO [StoreFileOpener-32b359af0da6a75b36c596a655fd4546-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a83ddb58162142efa1b15ac33111fcea 2024-11-13T16:26:26,184 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/a83ddb58162142efa1b15ac33111fcea 2024-11-13T16:26:26,184 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(1722): Closing 32b359af0da6a75b36c596a655fd4546, disabling compactions & flushes 2024-11-13T16:26:26,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. after waiting 0 ms 2024-11-13T16:26:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33401 {}] regionserver.HRegion(1676): Region close journal for 32b359af0da6a75b36c596a655fd4546: Waiting for close lock at 1731515186185Disabling compacts and flushes for region at 1731515186185Disabling writes for close at 1731515186185Writing region close event to WAL at 1731515186187 (+2 ms)Closed at 1731515186187 2024-11-13T16:26:26,247 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,247 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T16:26:26,247 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 32b359af0da6a75b36c596a655fd4546, disabling compactions & flushes 2024-11-13T16:26:26,247 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,247 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,247 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. after waiting 0 ms 2024-11-13T16:26:26,247 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,249 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/13ad8f0893ab48679eda3bbd561c1336, hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/656e67665f164aa998e51382836c6df7] to archive 2024-11-13T16:26:26,253 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T16:26:26,258 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/13ad8f0893ab48679eda3bbd561c1336 to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/13ad8f0893ab48679eda3bbd561c1336 2024-11-13T16:26:26,260 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/656e67665f164aa998e51382836c6df7 to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/656e67665f164aa998e51382836c6df7 2024-11-13T16:26:26,276 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/a83ddb58162142efa1b15ac33111fcea] to archive 2024-11-13T16:26:26,278 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T16:26:26,281 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/a83ddb58162142efa1b15ac33111fcea to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/a83ddb58162142efa1b15ac33111fcea 2024-11-13T16:26:26,293 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T16:26:26,294 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,294 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 32b359af0da6a75b36c596a655fd4546: Waiting for close lock at 1731515186247Running coprocessor pre-close hooks at 1731515186247Disabling compacts and flushes for region at 1731515186247Disabling writes for close at 1731515186247Writing region close event to WAL at 1731515186286 (+39 ms)Running coprocessor post-close hooks at 1731515186293 (+7 ms)Closed at 1731515186293 2024-11-13T16:26:26,294 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 32b359af0da6a75b36c596a655fd4546 move to f8f85b0bb1d8,33401,1731515163358 record at close sequenceid=12 2024-11-13T16:26:26,298 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,300 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=CLOSED 2024-11-13T16:26:26,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204 because future has completed 2024-11-13T16:26:26,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T16:26:26,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204 in 215 msec 2024-11-13T16:26:26,313 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE; state=CLOSED, location=f8f85b0bb1d8,33401,1731515163358; forceNewPlan=false, retain=false 2024-11-13T16:26:26,463 INFO [f8f85b0bb1d8:34015 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T16:26:26,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=OPENING, regionLocation=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:26,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE because future has completed 2024-11-13T16:26:26,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358}] 2024-11-13T16:26:26,626 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,627 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 32b359af0da6a75b36c596a655fd4546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:26,627 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,627 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:26,627 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,628 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,629 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,631 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf1 2024-11-13T16:26:26,631 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:26,640 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/63c27a0f95ed483c8dc979cee723ad87 2024-11-13T16:26:26,640 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:26,640 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,641 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf2 2024-11-13T16:26:26,642 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:26,652 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/42f69f18b99e49b9916b06a0852a9611 2024-11-13T16:26:26,652 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:26,652 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,653 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,655 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,656 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,656 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,657 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T16:26:26,659 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,661 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 32b359af0da6a75b36c596a655fd4546; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74769033, jitterRate=0.11414541304111481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T16:26:26,661 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,662 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 32b359af0da6a75b36c596a655fd4546: Running coprocessor pre-open hook at 1731515186628Writing region info on filesystem at 1731515186628Initializing all the Stores at 1731515186629 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515186629Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515186629Cleaning up temporary data from old regions at 1731515186656 (+27 ms)Running coprocessor post-open hooks at 1731515186661 (+5 ms)Region opened successfully at 1731515186662 (+1 ms) 2024-11-13T16:26:26,666 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., pid=12, masterSystemTime=1731515186622 2024-11-13T16:26:26,669 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,669 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,670 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=OPEN, openSeqNum=18, regionLocation=f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:26,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358 because future has completed 2024-11-13T16:26:26,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T16:26:26,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,33401,1731515163358 in 207 msec 2024-11-13T16:26:26,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, REOPEN/MOVE in 605 msec 2024-11-13T16:26:26,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T16:26:26,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:26:26,745 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server f8f85b0bb1d8,33401,1731515163358: testing ***** 2024-11-13T16:26:26,745 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T16:26:26,749 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T16:26:26,751 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T16:26:26,759 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T16:26:26,761 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T16:26:26,778 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 337350104 }, "NonHeapMemoryUsage": { "committed": 172032000, "init": 7667712, "max": -1, "used": 169437656 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "f8f85b0bb1d8", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10264, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 111, "ProcessCallTime_mean": 8, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 3, "ProcessCallTime_75th_percentile": 5, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 35, "ProcessCallTime_98th_percentile": 111, "ProcessCallTime_99th_percentile": 111, "ProcessCallTime_99.9th_percentile": 111, "ProcessCallTime_TimeRangeCount_0-1": 5, "ProcessCallTime_TimeRangeCount_1-3": 14, "ProcessCallTime_TimeRangeCount_3-10": 9, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "ProcessCallTime_TimeRangeCount_100-300": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 2, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 2, "QueueCallTime_99th_percentile": 2, "QueueCallTime_99.9th_percentile": 2, "QueueCallTime_TimeRangeCount_0-1": 30, "QueueCallTime_TimeRangeCount_1-3": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 111, "TotalCallTime_mean": 8, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 3, "TotalCallTime_75th_percentile": 5, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 35, "TotalCallTime_98th_percentile": 111, "TotalCallTime_99th_percentile": 111, "TotalCallTime_99.9th_percentile": 111, "TotalCallTime_TimeRangeCount_0-1": 5, "TotalCallTime_TimeRangeCount_1-3": 12, "TotalCallTime_TimeRangeCount_3-10": 11, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "TotalCallTime_TimeRangeCount_100-300": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5773 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "f8f85b0bb1d8", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:54878", "tag.serverName": "f8f85b0bb1d8,33401,1731515163358", "tag.clusterId": "f6d1291a-e193-41b7-8dec-6caf412160f7", "tag.Context": "regionserver", "tag.Hostname": "f8f85b0bb1d8", "regionCount": 2, "storeCount": 4, "hlogFileCount": 8, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 3789, "memStoreHeapSize": 6880, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731515163358, "averageRegionSize": 3789, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.8, "writeRequestRatePerSecond": 1.8, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 599040, "ByteBuffAllocatorHeapAllocationRatio": 0.35775729671400613, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 29, "totalRowActionRequestCount": 22, "readRequestCount": 11, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 11, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 17, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 4, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 4, "ScanTime_95th_percentile": 4, "ScanTime_98th_percentile": 4, "ScanTime_99th_percentile": 4, "ScanTime_99.9th_percentile": 4, "ScanTime_TimeRangeCount_0-1": 3, "ScanTime_TimeRangeCount_1-3": 1, "ScanTime_TimeRangeCount_3-10": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 16, "Put_min": 1, "Put_max": 7, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 14, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 468, "FlushTime_max": 468, "FlushTime_mean": 468, "FlushTime_25th_percentile": 468, "FlushTime_median": 468, "FlushTime_75th_percentile": 468, "FlushTime_90th_percentile": 468, "FlushTime_95th_percentile": 468, "FlushTime_98th_percentile": 468, "FlushTime_99th_percentile": 468, "FlushTime_99.9th_percentile": 468, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 64, "ScanSize_max": 1112, "ScanSize_mean": 902, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_10-100": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T16:26:26,784 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34015 {}] master.MasterRpcServices(700): f8f85b0bb1d8,33401,1731515163358 reported a fatal error: ***** ABORTING region server f8f85b0bb1d8,33401,1731515163358: testing ***** 2024-11-13T16:26:26,803 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f8f85b0bb1d8,33401,1731515163358' ***** 2024-11-13T16:26:26,803 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T16:26:26,804 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T16:26:26,804 INFO [RS:1;f8f85b0bb1d8:33401 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T16:26:26,804 INFO [RS:1;f8f85b0bb1d8:33401 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T16:26:26,804 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(3091): Received CLOSE for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,805 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(956): aborting server f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:26,805 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T16:26:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37847 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:40412 deadline: 1731515246805, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f8f85b0bb1d8 port=33401 startCode=1731515163358. As of locationSeqNum=12. 2024-11-13T16:26:26,805 INFO [RS:1;f8f85b0bb1d8:33401 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f8f85b0bb1d8:33401. 2024-11-13T16:26:26,805 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:26:26,806 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:26,806 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,37847,1731515163204, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,37847,1731515163204, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f8f85b0bb1d8 port=33401 startCode=1731515163358. As of locationSeqNum=12. 2024-11-13T16:26:26,806 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,37847,1731515163204, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f8f85b0bb1d8 port=33401 startCode=1731515163358. As of locationSeqNum=12. 2024-11-13T16:26:26,806 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,37847,1731515163204, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f8f85b0bb1d8 port=33401 startCode=1731515163358. As of locationSeqNum=12. 2024-11-13T16:26:26,806 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T16:26:26,806 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T16:26:26,806 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T16:26:26,806 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T16:26:26,807 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 32b359af0da6a75b36c596a655fd4546, disabling compactions & flushes 2024-11-13T16:26:26,807 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,807 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T16:26:26,807 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,807 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. after waiting 0 ms 2024-11-13T16:26:26,807 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,807 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T16:26:26,807 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 32b359af0da6a75b36c596a655fd4546=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.} 2024-11-13T16:26:26,808 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:26,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f8f85b0bb1d8,33401,1731515163358 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:26,810 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f8f85b0bb1d8,33401,1731515163358 aborting 2024-11-13T16:26:26,810 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f8f85b0bb1d8,33401,1731515163358 aborting 2024-11-13T16:26:26,810 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 from cache 2024-11-13T16:26:26,811 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T16:26:26,811 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T16:26:26,811 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T16:26:26,811 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T16:26:26,811 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T16:26:26,823 ERROR [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-13T16:26:26,823 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T16:26:26,824 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T16:26:26,824 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731515186811Running coprocessor pre-close hooks at 1731515186811Disabling compacts and flushes for region at 1731515186811Disabling writes for close at 1731515186811Writing region close event to WAL at 1731515186823 (+12 ms)Running coprocessor post-close hooks at 1731515186823Closed at 1731515186823 2024-11-13T16:26:26,824 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T16:26:26,824 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,824 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 32b359af0da6a75b36c596a655fd4546: Waiting for close lock at 1731515186806Running coprocessor pre-close hooks at 1731515186807 (+1 ms)Disabling compacts and flushes for region at 1731515186807Disabling writes for close at 1731515186807Writing region close event to WAL at 1731515186824 (+17 ms)Running coprocessor post-close hooks at 1731515186824Closed at 1731515186824 2024-11-13T16:26:26,824 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:26,825 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T16:26:26,825 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T16:26:26,846 INFO [regionserver/f8f85b0bb1d8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:26:26,914 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:26:26,915 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1] 2024-11-13T16:26:26,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f8f85b0bb1d8,33401,1731515163358 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:26,917 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f8f85b0bb1d8,33401,1731515163358 aborting 2024-11-13T16:26:26,917 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f8f85b0bb1d8,33401,1731515163358 aborting 2024-11-13T16:26:26,917 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 from cache 2024-11-13T16:26:27,008 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(976): stopping server f8f85b0bb1d8,33401,1731515163358; all regions closed. 2024-11-13T16:26:27,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741836_1012 (size=2747) 2024-11-13T16:26:27,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741836_1012 (size=2747) 2024-11-13T16:26:27,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741836_1012 (size=2747) 2024-11-13T16:26:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741834_1010 (size=1677) 2024-11-13T16:26:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741834_1010 (size=1677) 2024-11-13T16:26:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741834_1010 (size=1677) 2024-11-13T16:26:27,016 DEBUG [RS:1;f8f85b0bb1d8:33401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:27,016 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:26:27,016 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T16:26:27,016 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.ChoreService(370): Chore service for: regionserver/f8f85b0bb1d8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T16:26:27,016 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T16:26:27,016 INFO [regionserver/f8f85b0bb1d8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T16:26:27,017 INFO [RS:1;f8f85b0bb1d8:33401 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33401 2024-11-13T16:26:27,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f8f85b0bb1d8,33401,1731515163358 2024-11-13T16:26:27,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T16:26:27,032 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T16:26:27,033 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f8f85b0bb1d8,33401,1731515163358] 2024-11-13T16:26:27,048 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f8f85b0bb1d8,33401,1731515163358 already deleted, retry=false 2024-11-13T16:26:27,048 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f8f85b0bb1d8,33401,1731515163358 on f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:27,055 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,33401,1731515163358, splitWal=true, meta=true 2024-11-13T16:26:27,058 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for f8f85b0bb1d8,33401,1731515163358 (carryingMeta=true) f8f85b0bb1d8,33401,1731515163358/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@11c9cfba[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T16:26:27,059 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,33401,1731515163358, splitWal=true, meta=true 2024-11-13T16:26:27,060 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,33401,1731515163358, splitWal=true, meta=true, isMeta: true 2024-11-13T16:26:27,062 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting 2024-11-13T16:26:27,063 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,33401,1731515163358 WAL count=1, meta=true 2024-11-13T16:26:27,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta}] 2024-11-13T16:26:27,071 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:27,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, worker=f8f85b0bb1d8,37847,1731515163204}] 2024-11-13T16:26:27,123 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:26:27,124 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1] 2024-11-13T16:26:27,126 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:33401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f8f85b0bb1d8/172.17.0.3:33401 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:27,126 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:33401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f8f85b0bb1d8/172.17.0.3:33401 2024-11-13T16:26:27,127 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-13T16:26:27,127 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 from cache 2024-11-13T16:26:27,127 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:33401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f8f85b0bb1d8/172.17.0.3:33401 2024-11-13T16:26:27,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:26:27,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33401-0x101363260180002, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:26:27,141 INFO [RS:1;f8f85b0bb1d8:33401 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T16:26:27,141 INFO [RS:1;f8f85b0bb1d8:33401 {}] regionserver.HRegionServer(1031): Exiting; stopping=f8f85b0bb1d8,33401,1731515163358; zookeeper connection closed. 2024-11-13T16:26:27,142 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@73daa3bf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@73daa3bf 2024-11-13T16:26:27,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37847 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T16:26:27,264 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, size=2.7 K (2747bytes) 2024-11-13T16:26:27,265 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta 2024-11-13T16:26:27,266 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta after 0ms 2024-11-13T16:26:27,269 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:27,269 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta took 5ms 2024-11-13T16:26:27,279 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-13T16:26:27,280 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta so closing down 2024-11-13T16:26:27,281 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:27,288 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta.temp 2024-11-13T16:26:27,295 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta.temp 2024-11-13T16:26:27,295 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:27,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741892_1070 (size=2527) 2024-11-13T16:26:27,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741892_1070 (size=2527) 2024-11-13T16:26:27,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741892_1070 (size=2527) 2024-11-13T16:26:27,318 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:27,319 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta.temp to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T16:26:27,320 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 49 ms; skipped=1; WAL=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, size=2.7 K, length=2747, corrupted=false, cancelled=false 2024-11-13T16:26:27,320 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, journal: Splitting hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, size=2.7 K (2747bytes) at 1731515187265Finishing writing output for hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta so closing down at 1731515187281 (+16 ms)Creating recovered edits writer path=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta.temp at 1731515187295 (+14 ms)3 split writer threads finished at 1731515187295Closed recovered edits writer path=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731515187318 (+23 ms)Rename recovered edits hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta.temp to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731515187319 (+1 ms)Processed 16 edits across 1 Regions in 49 ms; skipped=1; WAL=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, size=2.7 K, length=2747, corrupted=false, cancelled=false at 1731515187320 (+1 ms) 2024-11-13T16:26:27,320 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta 2024-11-13T16:26:27,321 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T16:26:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T16:26:27,331 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs 2024-11-13T16:26:27,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T16:26:27,336 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, worker=f8f85b0bb1d8,37847,1731515163204 in 257 msec 2024-11-13T16:26:27,339 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:27,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T16:26:27,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C33401%2C1731515163358.meta.1731515166568.meta, worker=f8f85b0bb1d8,37847,1731515163204 in 275 msec 2024-11-13T16:26:27,345 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting dir is empty, no logs to split. 2024-11-13T16:26:27,345 INFO [PEWorker-1 {}] master.SplitWALManager(105): f8f85b0bb1d8,33401,1731515163358 WAL count=0, meta=true 2024-11-13T16:26:27,346 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if f8f85b0bb1d8,33401,1731515163358 WAL splitting is done? wals=0, meta=true 2024-11-13T16:26:27,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T16:26:27,350 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:26:27,352 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:26:27,433 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:26:27,435 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1] 2024-11-13T16:26:27,436 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:33401 this server is in the failed servers list 2024-11-13T16:26:27,437 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:33401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:33401 2024-11-13T16:26:27,437 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:33401 2024-11-13T16:26:27,437 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 from cache 2024-11-13T16:26:27,502 DEBUG [f8f85b0bb1d8:34015 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T16:26:27,503 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(204): Hosts are {f8f85b0bb1d8=0} racks are {/default-rack=0} 2024-11-13T16:26:27,503 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T16:26:27,503 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T16:26:27,503 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T16:26:27,503 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T16:26:27,503 INFO [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T16:26:27,503 INFO [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T16:26:27,503 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T16:26:27,504 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:27,505 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,40565,1731515163497, state=OPENING 2024-11-13T16:26:27,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:27,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:27,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:27,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:27,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:27,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:27,537 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T16:26:27,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,40565,1731515163497}] 2024-11-13T16:26:27,691 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T16:26:27,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37145, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T16:26:27,703 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T16:26:27,703 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:27,704 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T16:26:27,721 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f8f85b0bb1d8%2C40565%2C1731515163497.meta, suffix=.meta, logDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,40565,1731515163497, archiveDir=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs, maxLogs=32 2024-11-13T16:26:27,741 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,40565,1731515163497/f8f85b0bb1d8%2C40565%2C1731515163497.meta.1731515187722.meta, exclude list is [], retry=0 2024-11-13T16:26:27,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:27,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:27,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:27,760 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,40565,1731515163497/f8f85b0bb1d8%2C40565%2C1731515163497.meta.1731515187722.meta 2024-11-13T16:26:27,761 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:27,761 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:27,762 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T16:26:27,762 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T16:26:27,762 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T16:26:27,762 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T16:26:27,762 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:27,763 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T16:26:27,763 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T16:26:27,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T16:26:27,767 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T16:26:27,767 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:27,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:27,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T16:26:27,769 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T16:26:27,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:27,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:27,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T16:26:27,772 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T16:26:27,772 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:27,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:27,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T16:26:27,783 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T16:26:27,783 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:27,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T16:26:27,785 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T16:26:27,787 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740 2024-11-13T16:26:27,790 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740 2024-11-13T16:26:27,791 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T16:26:27,794 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:27,797 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T16:26:27,797 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-13T16:26:27,832 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/info/c830c09dc4554f208c3b454878261db9 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546./info:regioninfo/1731515186670/Put/seqid=0 2024-11-13T16:26:27,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741894_1072 (size=11177) 2024-11-13T16:26:27,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741894_1072 (size=11177) 2024-11-13T16:26:27,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741894_1072 (size=11177) 2024-11-13T16:26:27,944 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:26:27,945 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,40565,1731515163497, seqNum=-1] 2024-11-13T16:26:27,946 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:26:27,948 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:26:27,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40565 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.3:54812 deadline: 1731515247948, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:27,950 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,40565,1731515163497, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,40565,1731515163497, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,40565,1731515163497 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T16:26:27,950 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,40565,1731515163497, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on f8f85b0bb1d8,40565,1731515163497 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T16:26:27,950 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,40565,1731515163497, seqNum=-1 from cache 2024-11-13T16:26:28,257 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/info/c830c09dc4554f208c3b454878261db9 2024-11-13T16:26:28,288 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/ns/b35557a003bd4285a336dbf9cbcb6e93 is 43, key is default/ns:d/1731515167061/Put/seqid=0 2024-11-13T16:26:28,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741895_1073 (size=5153) 2024-11-13T16:26:28,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741895_1073 (size=5153) 2024-11-13T16:26:28,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741895_1073 (size=5153) 2024-11-13T16:26:28,304 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/ns/b35557a003bd4285a336dbf9cbcb6e93 2024-11-13T16:26:28,329 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/table/145bb6da042640a4bcecaf554fcf7886 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731515184127/Put/seqid=0 2024-11-13T16:26:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741896_1074 (size=5431) 2024-11-13T16:26:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741896_1074 (size=5431) 2024-11-13T16:26:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741896_1074 (size=5431) 2024-11-13T16:26:28,336 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/table/145bb6da042640a4bcecaf554fcf7886 2024-11-13T16:26:28,342 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/info/c830c09dc4554f208c3b454878261db9 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/info/c830c09dc4554f208c3b454878261db9 2024-11-13T16:26:28,349 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/info/c830c09dc4554f208c3b454878261db9, entries=36, sequenceid=18, filesize=10.9 K 2024-11-13T16:26:28,350 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/ns/b35557a003bd4285a336dbf9cbcb6e93 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/ns/b35557a003bd4285a336dbf9cbcb6e93 2024-11-13T16:26:28,357 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/ns/b35557a003bd4285a336dbf9cbcb6e93, entries=2, sequenceid=18, filesize=5.0 K 2024-11-13T16:26:28,359 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/table/145bb6da042640a4bcecaf554fcf7886 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/table/145bb6da042640a4bcecaf554fcf7886 2024-11-13T16:26:28,365 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/table/145bb6da042640a4bcecaf554fcf7886, entries=2, sequenceid=18, filesize=5.3 K 2024-11-13T16:26:28,365 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 568ms, sequenceid=18, compaction requested=false; wal=null 2024-11-13T16:26:28,366 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T16:26:28,367 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T16:26:28,367 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T16:26:28,368 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T16:26:28,370 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T16:26:28,373 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-13T16:26:28,374 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60074465, jitterRate=-0.10482071340084076}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T16:26:28,374 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T16:26:28,375 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731515187763Writing region info on filesystem at 1731515187763Initializing all the Stores at 1731515187765 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515187765Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515187765Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515187765Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731515187765Obtaining lock to block concurrent updates at 1731515187797 (+32 ms)Preparing flush snapshotting stores in 1588230740 at 1731515187797Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731515187798 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731515187798Flushing 1588230740/info: creating writer at 1731515187798Flushing 1588230740/info: appending metadata at 1731515187831 (+33 ms)Flushing 1588230740/info: closing flushed file at 1731515187831Flushing 1588230740/ns: creating writer at 1731515188265 (+434 ms)Flushing 1588230740/ns: appending metadata at 1731515188287 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731515188287Flushing 1588230740/table: creating writer at 1731515188311 (+24 ms)Flushing 1588230740/table: appending metadata at 1731515188328 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731515188328Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d3df45c: reopening flushed file at 1731515188341 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16c39140: reopening flushed file at 1731515188349 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f6220e9: reopening flushed file at 1731515188358 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 568ms, sequenceid=18, compaction requested=false; wal=null at 1731515188365 (+7 ms)Cleaning up temporary data from old regions at 1731515188367 (+2 ms)Running coprocessor post-open hooks at 1731515188374 (+7 ms)Region opened successfully at 1731515188375 (+1 ms) 2024-11-13T16:26:28,376 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731515187691 2024-11-13T16:26:28,379 DEBUG [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T16:26:28,379 INFO [RS_OPEN_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T16:26:28,380 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:28,381 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f8f85b0bb1d8,40565,1731515163497, state=OPEN 2024-11-13T16:26:28,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:28,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:28,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T16:26:28,390 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:28,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:28,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:28,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T16:26:28,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T16:26:28,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f8f85b0bb1d8,40565,1731515163497 in 853 msec 2024-11-13T16:26:28,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T16:26:28,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0450 sec 2024-11-13T16:26:28,399 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): f8f85b0bb1d8,33401,1731515163358 had 2 regions 2024-11-13T16:26:28,400 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f8f85b0bb1d8,33401,1731515163358, splitWal=true, meta=true, isMeta: false 2024-11-13T16:26:28,402 INFO [PEWorker-3 {}] master.SplitWALManager(105): f8f85b0bb1d8,33401,1731515163358 WAL count=1, meta=false 2024-11-13T16:26:28,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963}] 2024-11-13T16:26:28,405 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:28,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, worker=f8f85b0bb1d8,40565,1731515163497}] 2024-11-13T16:26:28,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40565 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-13T16:26:28,584 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, size=1.6 K (1677bytes) 2024-11-13T16:26:28,584 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963 2024-11-13T16:26:28,584 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963 after 0ms 2024-11-13T16:26:28,587 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:28,587 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963 took 3ms 2024-11-13T16:26:28,589 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 32b359af0da6a75b36c596a655fd4546: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T16:26:28,590 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963 so closing down 2024-11-13T16:26:28,590 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:28,590 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:28,590 INFO [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-11-13T16:26:28,590 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, journal: Splitting hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, size=1.6 K (1677bytes) at 1731515188584Finishing writing output for hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963 so closing down at 1731515188590 (+6 ms)3 split writer threads finished at 1731515188590Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1731515188590 2024-11-13T16:26:28,590 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963 2024-11-13T16:26:28,590 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f8f85b0bb1d8:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-13T16:26:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34015 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-13T16:26:28,594 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting/f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963 to hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs 2024-11-13T16:26:28,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-13T16:26:28,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, worker=f8f85b0bb1d8,40565,1731515163497 in 185 msec 2024-11-13T16:26:28,598 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:28,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-13T16:26:28,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure f8f85b0bb1d8%2C33401%2C1731515163358.1731515165963, worker=f8f85b0bb1d8,40565,1731515163497 in 196 msec 2024-11-13T16:26:28,603 INFO [PEWorker-3 {}] master.SplitLogManager(171): hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/WALs/f8f85b0bb1d8,33401,1731515163358-splitting dir is empty, no logs to split. 2024-11-13T16:26:28,603 INFO [PEWorker-3 {}] master.SplitWALManager(105): f8f85b0bb1d8,33401,1731515163358 WAL count=0, meta=false 2024-11-13T16:26:28,603 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if f8f85b0bb1d8,33401,1731515163358 WAL splitting is done? wals=0, meta=false 2024-11-13T16:26:28,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN}] 2024-11-13T16:26:28,607 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN 2024-11-13T16:26:28,608 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T16:26:28,759 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(204): Hosts are {f8f85b0bb1d8=0} racks are {/default-rack=0} 2024-11-13T16:26:28,759 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T16:26:28,759 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T16:26:28,759 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T16:26:28,759 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T16:26:28,759 INFO [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T16:26:28,759 INFO [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T16:26:28,759 DEBUG [f8f85b0bb1d8:34015 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T16:26:28,760 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=OPENING, regionLocation=f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:28,761 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f8f85b0bb1d8:33401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:33401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:28,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1, error=java.net.ConnectException: Call to address=f8f85b0bb1d8:33401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:33401 2024-11-13T16:26:28,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T16:26:28,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=-1 from cache 2024-11-13T16:26:28,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.FailedServers(52): Added failed server with address f8f85b0bb1d8:33401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f8f85b0bb1d8/172.17.0.3:33401 2024-11-13T16:26:28,864 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:26:28,864 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,40565,1731515163497, seqNum=-1] 2024-11-13T16:26:28,864 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T16:26:28,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59383, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T16:26:28,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN because future has completed 2024-11-13T16:26:28,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204}] 2024-11-13T16:26:28,964 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T16:26:28,965 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f8f85b0bb1d8,40565,1731515163497, seqNum=-1] 2024-11-13T16:26:29,026 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:29,027 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 32b359af0da6a75b36c596a655fd4546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:29,027 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,027 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:29,028 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,028 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,030 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,032 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf1 2024-11-13T16:26:29,032 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:29,040 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf1/63c27a0f95ed483c8dc979cee723ad87 2024-11-13T16:26:29,040 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:29,040 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,042 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32b359af0da6a75b36c596a655fd4546 columnFamilyName cf2 2024-11-13T16:26:29,042 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:29,050 DEBUG [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/cf2/42f69f18b99e49b9916b06a0852a9611 2024-11-13T16:26:29,050 INFO [StoreOpener-32b359af0da6a75b36c596a655fd4546-1 {}] regionserver.HStore(327): Store=32b359af0da6a75b36c596a655fd4546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:29,050 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,051 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,053 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,054 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,054 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,055 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T16:26:29,057 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,058 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 32b359af0da6a75b36c596a655fd4546; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59493552, jitterRate=-0.11347699165344238}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T16:26:29,058 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:29,059 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 32b359af0da6a75b36c596a655fd4546: Running coprocessor pre-open hook at 1731515189028Writing region info on filesystem at 1731515189028Initializing all the Stores at 1731515189030 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515189030Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515189030Cleaning up temporary data from old regions at 1731515189054 (+24 ms)Running coprocessor post-open hooks at 1731515189058 (+4 ms)Region opened successfully at 1731515189059 (+1 ms) 2024-11-13T16:26:29,060 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., pid=21, masterSystemTime=1731515189021 2024-11-13T16:26:29,062 DEBUG [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:29,063 INFO [RS_OPEN_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:29,063 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=32b359af0da6a75b36c596a655fd4546, regionState=OPEN, openSeqNum=18, regionLocation=f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:29,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204 because future has completed 2024-11-13T16:26:29,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T16:26:29,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 32b359af0da6a75b36c596a655fd4546, server=f8f85b0bb1d8,37847,1731515163204 in 202 msec 2024-11-13T16:26:29,083 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f8f85b0bb1d8:33401 this server is in the failed servers list 2024-11-13T16:26:29,084 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f8f85b0bb1d8:33401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:33401 2024-11-13T16:26:29,084 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f8f85b0bb1d8:33401 2024-11-13T16:26:29,084 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,33401,1731515163358, seqNum=12 from cache 2024-11-13T16:26:29,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-13T16:26:29,086 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server f8f85b0bb1d8,33401,1731515163358 after splitting done 2024-11-13T16:26:29,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=32b359af0da6a75b36c596a655fd4546, ASSIGN in 471 msec 2024-11-13T16:26:29,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure f8f85b0bb1d8,33401,1731515163358, splitWal=true, meta=true in 2.0360 sec 2024-11-13T16:26:29,308 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546., hostname=f8f85b0bb1d8,37847,1731515163204, seqNum=18] 2024-11-13T16:26:29,327 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=402 (was 409), OpenFileDescriptor=1028 (was 1010) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=914 (was 958), ProcessCount=11 (was 11), AvailableMemoryMB=2793 (was 2941) 2024-11-13T16:26:29,330 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1028 is superior to 1024 2024-11-13T16:26:29,345 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=402, OpenFileDescriptor=1028, MaxFileDescriptor=1048576, SystemLoadAverage=914, ProcessCount=11, AvailableMemoryMB=2793 2024-11-13T16:26:29,345 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1028 is superior to 1024 2024-11-13T16:26:29,360 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:29,362 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:29,363 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:29,366 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98237911, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-98237911, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:29,381 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98237911/hregion-98237911.1731515189366, exclude list is [], retry=0 2024-11-13T16:26:29,384 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:29,384 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:29,385 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:29,387 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98237911/hregion-98237911.1731515189366 2024-11-13T16:26:29,387 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:29,387 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5effbf3111785d697cd6b8dac1b53e4c, NAME => 'testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:29,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741898_1076 (size=67) 2024-11-13T16:26:29,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741898_1076 (size=67) 2024-11-13T16:26:29,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741898_1076 (size=67) 2024-11-13T16:26:29,402 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:29,403 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,405 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName a 2024-11-13T16:26:29,405 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:29,406 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:29,406 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,407 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName b 2024-11-13T16:26:29,408 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:29,408 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:29,408 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,411 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName c 2024-11-13T16:26:29,411 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:29,411 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:29,411 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,412 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,413 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,414 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,414 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,415 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:29,418 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,421 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:29,421 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5effbf3111785d697cd6b8dac1b53e4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59608258, jitterRate=-0.1117677390575409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:29,422 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5effbf3111785d697cd6b8dac1b53e4c: Writing region info on filesystem at 1731515189402Initializing all the Stores at 1731515189402Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515189402Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515189403 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515189403Cleaning up temporary data from old regions at 1731515189414 (+11 ms)Region opened successfully at 1731515189422 (+8 ms) 2024-11-13T16:26:29,422 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5effbf3111785d697cd6b8dac1b53e4c, disabling compactions & flushes 2024-11-13T16:26:29,422 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. 2024-11-13T16:26:29,422 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. 2024-11-13T16:26:29,422 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. after waiting 0 ms 2024-11-13T16:26:29,422 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. 2024-11-13T16:26:29,424 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. 2024-11-13T16:26:29,425 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5effbf3111785d697cd6b8dac1b53e4c: Waiting for close lock at 1731515189422Disabling compacts and flushes for region at 1731515189422Disabling writes for close at 1731515189422Writing region close event to WAL at 1731515189424 (+2 ms)Closed at 1731515189424 2024-11-13T16:26:29,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741897_1075 (size=93) 2024-11-13T16:26:29,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741897_1075 (size=93) 2024-11-13T16:26:29,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741897_1075 (size=93) 2024-11-13T16:26:29,433 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:29,433 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98237911:(num 1731515189366) 2024-11-13T16:26:29,434 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:29,443 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:29,460 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444, exclude list is [], retry=0 2024-11-13T16:26:29,463 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:29,464 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:29,465 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:29,470 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444 2024-11-13T16:26:29,471 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:29,471 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5effbf3111785d697cd6b8dac1b53e4c, NAME => 'testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:29,471 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:29,471 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,471 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,473 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,475 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName a 2024-11-13T16:26:29,475 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:29,476 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:29,476 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,477 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName b 2024-11-13T16:26:29,477 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:29,478 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:29,478 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,479 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName c 2024-11-13T16:26:29,486 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:29,488 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:29,488 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,489 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,490 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,492 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,492 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,493 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:29,495 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:29,496 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5effbf3111785d697cd6b8dac1b53e4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64453865, jitterRate=-0.039562568068504333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:29,497 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5effbf3111785d697cd6b8dac1b53e4c: Writing region info on filesystem at 1731515189472Initializing all the Stores at 1731515189473 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515189473Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515189473Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515189473Cleaning up temporary data from old regions at 1731515189492 (+19 ms)Region opened successfully at 1731515189497 (+5 ms) 2024-11-13T16:26:29,548 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5effbf3111785d697cd6b8dac1b53e4c 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T16:26:29,578 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/a/4721fe1581c84858a535cfebe150bd2f is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731515189497/Put/seqid=0 2024-11-13T16:26:29,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741900_1078 (size=5958) 2024-11-13T16:26:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741900_1078 (size=5958) 2024-11-13T16:26:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741900_1078 (size=5958) 2024-11-13T16:26:29,602 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T16:26:29,996 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/a/4721fe1581c84858a535cfebe150bd2f 2024-11-13T16:26:30,020 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/b/17885eb3cbce487fb6a148c447e1d009 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731515189517/Put/seqid=0 2024-11-13T16:26:30,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741901_1079 (size=5958) 2024-11-13T16:26:30,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741901_1079 (size=5958) 2024-11-13T16:26:30,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741901_1079 (size=5958) 2024-11-13T16:26:30,039 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/b/17885eb3cbce487fb6a148c447e1d009 2024-11-13T16:26:30,066 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/c/a6de94bd656c4c43bb5bb9d7b6fb36a0 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731515189534/Put/seqid=0 2024-11-13T16:26:30,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741902_1080 (size=5958) 2024-11-13T16:26:30,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741902_1080 (size=5958) 2024-11-13T16:26:30,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741902_1080 (size=5958) 2024-11-13T16:26:30,089 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/c/a6de94bd656c4c43bb5bb9d7b6fb36a0 2024-11-13T16:26:30,097 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/a/4721fe1581c84858a535cfebe150bd2f as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/a/4721fe1581c84858a535cfebe150bd2f 2024-11-13T16:26:30,114 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/a/4721fe1581c84858a535cfebe150bd2f, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T16:26:30,117 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/b/17885eb3cbce487fb6a148c447e1d009 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/b/17885eb3cbce487fb6a148c447e1d009 2024-11-13T16:26:30,123 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/b/17885eb3cbce487fb6a148c447e1d009, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T16:26:30,124 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/c/a6de94bd656c4c43bb5bb9d7b6fb36a0 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/c/a6de94bd656c4c43bb5bb9d7b6fb36a0 2024-11-13T16:26:30,131 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/c/a6de94bd656c4c43bb5bb9d7b6fb36a0, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T16:26:30,132 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 5effbf3111785d697cd6b8dac1b53e4c in 584ms, sequenceid=33, compaction requested=false 2024-11-13T16:26:30,132 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5effbf3111785d697cd6b8dac1b53e4c: 2024-11-13T16:26:30,133 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5effbf3111785d697cd6b8dac1b53e4c, disabling compactions & flushes 2024-11-13T16:26:30,133 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. 2024-11-13T16:26:30,133 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. 2024-11-13T16:26:30,133 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. after waiting 0 ms 2024-11-13T16:26:30,133 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. 2024-11-13T16:26:30,134 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. 2024-11-13T16:26:30,134 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5effbf3111785d697cd6b8dac1b53e4c: Waiting for close lock at 1731515190133Disabling compacts and flushes for region at 1731515190133Disabling writes for close at 1731515190133Writing region close event to WAL at 1731515190134 (+1 ms)Closed at 1731515190134 2024-11-13T16:26:30,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741899_1077 (size=2873) 2024-11-13T16:26:30,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741899_1077 (size=2873) 2024-11-13T16:26:30,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741899_1077 (size=2873) 2024-11-13T16:26:30,141 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/b/17885eb3cbce487fb6a148c447e1d009 to hdfs://localhost:40955/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/b/17885eb3cbce487fb6a148c447e1d009 2024-11-13T16:26:30,159 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444, size=2.8 K (2873bytes) 2024-11-13T16:26:30,159 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444 2024-11-13T16:26:30,159 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444 after 0ms 2024-11-13T16:26:30,164 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:30,164 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444 took 6ms 2024-11-13T16:26:30,167 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444 so closing down 2024-11-13T16:26:30,167 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:30,168 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731515189444.temp 2024-11-13T16:26:30,170 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000003-wal.1731515189444.temp 2024-11-13T16:26:30,170 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:30,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741903_1081 (size=2312) 2024-11-13T16:26:30,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741903_1081 (size=2312) 2024-11-13T16:26:30,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741903_1081 (size=2312) 2024-11-13T16:26:30,181 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000003-wal.1731515189444.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:30,183 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000003-wal.1731515189444.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000032 2024-11-13T16:26:30,183 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-11-13T16:26:30,183 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444, journal: Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444, size=2.8 K (2873bytes) at 1731515190159Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444 so closing down at 1731515190167 (+8 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000003-wal.1731515189444.temp at 1731515190170 (+3 ms)3 split writer threads finished at 1731515190170Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000003-wal.1731515189444.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731515190181 (+11 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000003-wal.1731515189444.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000032 at 1731515190183 (+2 ms)Processed 32 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1731515190183 2024-11-13T16:26:30,185 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515189444 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515189444 2024-11-13T16:26:30,186 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000032 2024-11-13T16:26:30,186 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:30,188 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:30,204 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515190188, exclude list is [], retry=0 2024-11-13T16:26:30,210 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:30,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:30,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:30,231 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515190188 2024-11-13T16:26:30,231 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:30,232 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5effbf3111785d697cd6b8dac1b53e4c, NAME => 'testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:30,232 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:30,232 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,232 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,239 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,240 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName a 2024-11-13T16:26:30,240 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,248 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/a/4721fe1581c84858a535cfebe150bd2f 2024-11-13T16:26:30,248 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,249 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,251 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName b 2024-11-13T16:26:30,251 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,252 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,252 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,253 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5effbf3111785d697cd6b8dac1b53e4c columnFamilyName c 2024-11-13T16:26:30,253 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,260 DEBUG [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/c/a6de94bd656c4c43bb5bb9d7b6fb36a0 2024-11-13T16:26:30,260 INFO [StoreOpener-5effbf3111785d697cd6b8dac1b53e4c-1 {}] regionserver.HStore(327): Store=5effbf3111785d697cd6b8dac1b53e4c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,260 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,261 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,264 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000032 2024-11-13T16:26:30,268 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:30,270 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000032 2024-11-13T16:26:30,270 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5effbf3111785d697cd6b8dac1b53e4c 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T16:26:30,287 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/b/c1dab7d2bc4c4d8188a6cbeec3600905 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731515189517/Put/seqid=0 2024-11-13T16:26:30,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741905_1083 (size=5958) 2024-11-13T16:26:30,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741905_1083 (size=5958) 2024-11-13T16:26:30,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741905_1083 (size=5958) 2024-11-13T16:26:30,296 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/b/c1dab7d2bc4c4d8188a6cbeec3600905 2024-11-13T16:26:30,301 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/.tmp/b/c1dab7d2bc4c4d8188a6cbeec3600905 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/b/c1dab7d2bc4c4d8188a6cbeec3600905 2024-11-13T16:26:30,308 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/b/c1dab7d2bc4c4d8188a6cbeec3600905, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T16:26:30,308 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 5effbf3111785d697cd6b8dac1b53e4c in 38ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T16:26:30,309 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/0000000000000000032 2024-11-13T16:26:30,310 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,310 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,311 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:30,312 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5effbf3111785d697cd6b8dac1b53e4c 2024-11-13T16:26:30,315 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/5effbf3111785d697cd6b8dac1b53e4c/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T16:26:30,316 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5effbf3111785d697cd6b8dac1b53e4c; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65959447, jitterRate=-0.01712764799594879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:30,317 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5effbf3111785d697cd6b8dac1b53e4c: Writing region info on filesystem at 1731515190232Initializing all the Stores at 1731515190235 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190235Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190238 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190238Obtaining lock to block concurrent updates at 1731515190270 (+32 ms)Preparing flush snapshotting stores in 5effbf3111785d697cd6b8dac1b53e4c at 1731515190270Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731515190271 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731515189361.5effbf3111785d697cd6b8dac1b53e4c. at 1731515190271Flushing 5effbf3111785d697cd6b8dac1b53e4c/b: creating writer at 1731515190271Flushing 5effbf3111785d697cd6b8dac1b53e4c/b: appending metadata at 1731515190287 (+16 ms)Flushing 5effbf3111785d697cd6b8dac1b53e4c/b: closing flushed file at 1731515190287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61a1f2d8: reopening flushed file at 1731515190301 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 5effbf3111785d697cd6b8dac1b53e4c in 38ms, sequenceid=32, compaction requested=false; wal=null at 1731515190308 (+7 ms)Cleaning up temporary data from old regions at 1731515190310 (+2 ms)Region opened successfully at 1731515190317 (+7 ms) 2024-11-13T16:26:30,335 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T16:26:30,336 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T16:26:30,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=412 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:40020 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:34222 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:48678 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:44550 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:51828 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:40172 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1104 (was 1028) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=914 (was 914), ProcessCount=11 (was 11), AvailableMemoryMB=2729 (was 2793) 2024-11-13T16:26:30,344 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1104 is superior to 1024 2024-11-13T16:26:30,360 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=412, OpenFileDescriptor=1104, MaxFileDescriptor=1048576, SystemLoadAverage=914, ProcessCount=11, AvailableMemoryMB=2727 2024-11-13T16:26:30,361 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1104 is superior to 1024 2024-11-13T16:26:30,377 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:30,380 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:30,381 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:30,385 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82181216, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-82181216, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:30,404 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82181216/hregion-82181216.1731515190386, exclude list is [], retry=0 2024-11-13T16:26:30,408 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:30,409 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:30,409 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:30,419 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82181216/hregion-82181216.1731515190386 2024-11-13T16:26:30,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:30,423 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 15bba1713cba209424a937d5093fd77e, NAME => 'testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:30,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741907_1085 (size=68) 2024-11-13T16:26:30,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741907_1085 (size=68) 2024-11-13T16:26:30,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741907_1085 (size=68) 2024-11-13T16:26:30,444 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:30,445 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,447 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName a 2024-11-13T16:26:30,447 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,448 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,448 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,450 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName b 2024-11-13T16:26:30,450 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,451 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,451 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,453 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName c 2024-11-13T16:26:30,453 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,454 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,454 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,455 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,457 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,457 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,457 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:30,458 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,461 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:30,461 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 15bba1713cba209424a937d5093fd77e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69460300, jitterRate=0.03503912687301636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:30,462 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 15bba1713cba209424a937d5093fd77e: Writing region info on filesystem at 1731515190444Initializing all the Stores at 1731515190444Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190445 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190445Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190445Cleaning up temporary data from old regions at 1731515190457 (+12 ms)Region opened successfully at 1731515190462 (+5 ms) 2024-11-13T16:26:30,462 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 15bba1713cba209424a937d5093fd77e, disabling compactions & flushes 2024-11-13T16:26:30,462 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:30,462 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:30,462 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. after waiting 0 ms 2024-11-13T16:26:30,462 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:30,463 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:30,463 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 15bba1713cba209424a937d5093fd77e: Waiting for close lock at 1731515190462Disabling compacts and flushes for region at 1731515190462Disabling writes for close at 1731515190462Writing region close event to WAL at 1731515190463 (+1 ms)Closed at 1731515190463 2024-11-13T16:26:30,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741906_1084 (size=93) 2024-11-13T16:26:30,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741906_1084 (size=93) 2024-11-13T16:26:30,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741906_1084 (size=93) 2024-11-13T16:26:30,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:30,470 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82181216:(num 1731515190386) 2024-11-13T16:26:30,470 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:30,473 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:30,490 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473, exclude list is [], retry=0 2024-11-13T16:26:30,493 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:30,493 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:30,493 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:30,495 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473 2024-11-13T16:26:30,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:30,567 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 15bba1713cba209424a937d5093fd77e, NAME => 'testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:30,569 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,569 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:30,570 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,570 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,572 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,573 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName a 2024-11-13T16:26:30,573 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,574 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,575 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,576 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName b 2024-11-13T16:26:30,576 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,576 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,576 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,577 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName c 2024-11-13T16:26:30,577 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:30,578 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:30,578 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,579 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,580 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,581 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,581 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,582 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:30,583 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,584 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 15bba1713cba209424a937d5093fd77e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74851694, jitterRate=0.11537715792655945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:30,584 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:30,585 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 15bba1713cba209424a937d5093fd77e: Running coprocessor pre-open hook at 1731515190570Writing region info on filesystem at 1731515190570Initializing all the Stores at 1731515190571 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190571Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190571Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515190571Cleaning up temporary data from old regions at 1731515190581 (+10 ms)Running coprocessor post-open hooks at 1731515190584 (+3 ms)Region opened successfully at 1731515190584 2024-11-13T16:26:30,608 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 15bba1713cba209424a937d5093fd77e 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T16:26:30,609 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:31,095 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T16:26:31,610 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:32,611 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:32,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T16:26:32,686 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:32,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T16:26:32,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:32,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T16:26:32,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:33,612 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:34,532 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T16:26:34,612 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:35,613 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:36,614 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:37,615 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:38,616 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:39,618 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 15bba1713cba209424a937d5093fd77e/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:39,620 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 15bba1713cba209424a937d5093fd77e: 2024-11-13T16:26:39,620 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:39,631 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 15bba1713cba209424a937d5093fd77e: 2024-11-13T16:26:39,631 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T16:26:39,631 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 15bba1713cba209424a937d5093fd77e, disabling compactions & flushes 2024-11-13T16:26:39,631 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:39,631 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:39,631 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. after waiting 0 ms 2024-11-13T16:26:39,631 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:39,632 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:39,632 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. 2024-11-13T16:26:39,632 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 15bba1713cba209424a937d5093fd77e: Waiting for close lock at 1731515199631Running coprocessor pre-close hooks at 1731515199631Disabling compacts and flushes for region at 1731515199631Disabling writes for close at 1731515199631Writing region close event to WAL at 1731515199632 (+1 ms)Running coprocessor post-close hooks at 1731515199632Closed at 1731515199632 2024-11-13T16:26:39,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741908_1086 (size=2398) 2024-11-13T16:26:39,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741908_1086 (size=2398) 2024-11-13T16:26:39,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741908_1086 (size=2398) 2024-11-13T16:26:39,654 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473, size=2.3 K (2398bytes) 2024-11-13T16:26:39,654 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473 2024-11-13T16:26:39,654 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473 after 0ms 2024-11-13T16:26:39,658 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:39,658 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473 took 5ms 2024-11-13T16:26:39,663 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473 so closing down 2024-11-13T16:26:39,663 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:39,667 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731515190473.temp 2024-11-13T16:26:39,668 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000004-wal.1731515190473.temp 2024-11-13T16:26:39,669 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:39,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741909_1087 (size=1672) 2024-11-13T16:26:39,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741909_1087 (size=1672) 2024-11-13T16:26:39,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741909_1087 (size=1672) 2024-11-13T16:26:39,683 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000004-wal.1731515190473.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:39,684 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000004-wal.1731515190473.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000026 2024-11-13T16:26:39,685 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 23 ms; skipped=3; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-11-13T16:26:39,685 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473, journal: Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473, size=2.3 K (2398bytes) at 1731515199654Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473 so closing down at 1731515199663 (+9 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000004-wal.1731515190473.temp at 1731515199668 (+5 ms)3 split writer threads finished at 1731515199669 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000004-wal.1731515190473.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731515199683 (+14 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000004-wal.1731515190473.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000026 at 1731515199684 (+1 ms)Processed 23 edits across 1 Regions in 23 ms; skipped=3; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1731515199685 (+1 ms) 2024-11-13T16:26:39,687 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515190473 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515190473 2024-11-13T16:26:39,688 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000026 2024-11-13T16:26:39,688 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:39,692 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:39,707 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515199693, exclude list is [], retry=0 2024-11-13T16:26:39,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:39,710 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:39,711 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:39,713 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515199693 2024-11-13T16:26:39,714 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:39,714 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 15bba1713cba209424a937d5093fd77e, NAME => 'testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:39,715 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,715 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:39,715 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,715 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,717 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,718 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName a 2024-11-13T16:26:39,718 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:39,719 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:39,719 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,720 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName b 2024-11-13T16:26:39,720 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:39,720 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:39,721 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,721 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15bba1713cba209424a937d5093fd77e columnFamilyName c 2024-11-13T16:26:39,722 DEBUG [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:39,722 INFO [StoreOpener-15bba1713cba209424a937d5093fd77e-1 {}] regionserver.HStore(327): Store=15bba1713cba209424a937d5093fd77e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:39,722 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,723 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,724 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,725 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000026 2024-11-13T16:26:39,727 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:39,728 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000026 2024-11-13T16:26:39,728 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 15bba1713cba209424a937d5093fd77e 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T16:26:39,745 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/a/4521ace9453f481caceac2d233567009 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731515199624/Put/seqid=0 2024-11-13T16:26:39,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741911_1089 (size=5523) 2024-11-13T16:26:39,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741911_1089 (size=5523) 2024-11-13T16:26:39,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741911_1089 (size=5523) 2024-11-13T16:26:39,767 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/a/4521ace9453f481caceac2d233567009 2024-11-13T16:26:39,791 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/b/83dd2c3f33c144569a1ba629eb8405bc is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731515199620/Put/seqid=0 2024-11-13T16:26:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741912_1090 (size=5524) 2024-11-13T16:26:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741912_1090 (size=5524) 2024-11-13T16:26:39,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741912_1090 (size=5524) 2024-11-13T16:26:39,799 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/b/83dd2c3f33c144569a1ba629eb8405bc 2024-11-13T16:26:39,820 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/c/6cae625a3c9643b5bc5818afb347e8e9 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731515199622/Put/seqid=0 2024-11-13T16:26:39,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741913_1091 (size=5457) 2024-11-13T16:26:39,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741913_1091 (size=5457) 2024-11-13T16:26:39,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741913_1091 (size=5457) 2024-11-13T16:26:39,826 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/c/6cae625a3c9643b5bc5818afb347e8e9 2024-11-13T16:26:39,833 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/a/4521ace9453f481caceac2d233567009 as hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/a/4521ace9453f481caceac2d233567009 2024-11-13T16:26:39,842 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/a/4521ace9453f481caceac2d233567009, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T16:26:39,843 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/b/83dd2c3f33c144569a1ba629eb8405bc as hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/b/83dd2c3f33c144569a1ba629eb8405bc 2024-11-13T16:26:39,850 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/b/83dd2c3f33c144569a1ba629eb8405bc, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T16:26:39,853 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/.tmp/c/6cae625a3c9643b5bc5818afb347e8e9 as hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/c/6cae625a3c9643b5bc5818afb347e8e9 2024-11-13T16:26:39,864 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/c/6cae625a3c9643b5bc5818afb347e8e9, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T16:26:39,864 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 15bba1713cba209424a937d5093fd77e in 136ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T16:26:39,866 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/0000000000000000026 2024-11-13T16:26:39,868 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,868 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,869 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:39,870 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,873 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsAfterAbortingFlush/15bba1713cba209424a937d5093fd77e/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T16:26:39,874 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 15bba1713cba209424a937d5093fd77e; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74458862, jitterRate=0.10952350497245789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:39,874 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15bba1713cba209424a937d5093fd77e 2024-11-13T16:26:39,875 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 15bba1713cba209424a937d5093fd77e: Running coprocessor pre-open hook at 1731515199715Writing region info on filesystem at 1731515199715Initializing all the Stores at 1731515199716 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515199716Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515199716Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515199716Obtaining lock to block concurrent updates at 1731515199728 (+12 ms)Preparing flush snapshotting stores in 15bba1713cba209424a937d5093fd77e at 1731515199728Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731515199728Flushing stores of testReplayEditsAfterAbortingFlush,,1731515190378.15bba1713cba209424a937d5093fd77e. at 1731515199728Flushing 15bba1713cba209424a937d5093fd77e/a: creating writer at 1731515199729 (+1 ms)Flushing 15bba1713cba209424a937d5093fd77e/a: appending metadata at 1731515199745 (+16 ms)Flushing 15bba1713cba209424a937d5093fd77e/a: closing flushed file at 1731515199745Flushing 15bba1713cba209424a937d5093fd77e/b: creating writer at 1731515199773 (+28 ms)Flushing 15bba1713cba209424a937d5093fd77e/b: appending metadata at 1731515199791 (+18 ms)Flushing 15bba1713cba209424a937d5093fd77e/b: closing flushed file at 1731515199791Flushing 15bba1713cba209424a937d5093fd77e/c: creating writer at 1731515199805 (+14 ms)Flushing 15bba1713cba209424a937d5093fd77e/c: appending metadata at 1731515199819 (+14 ms)Flushing 15bba1713cba209424a937d5093fd77e/c: closing flushed file at 1731515199819Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c65366c: reopening flushed file at 1731515199832 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f005a34: reopening flushed file at 1731515199842 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c946721: reopening flushed file at 1731515199850 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 15bba1713cba209424a937d5093fd77e in 136ms, sequenceid=26, compaction requested=false; wal=null at 1731515199864 (+14 ms)Cleaning up temporary data from old regions at 1731515199868 (+4 ms)Running coprocessor post-open hooks at 1731515199874 (+6 ms)Region opened successfully at 1731515199875 (+1 ms) 2024-11-13T16:26:39,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=418 (was 412) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58732 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37086 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58740 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37052 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58772 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58714 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1174 (was 1104) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=788 (was 914), ProcessCount=11 (was 11), AvailableMemoryMB=2654 (was 2727) 2024-11-13T16:26:39,896 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1174 is superior to 1024 2024-11-13T16:26:39,907 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=418, OpenFileDescriptor=1174, MaxFileDescriptor=1048576, SystemLoadAverage=788, ProcessCount=11, AvailableMemoryMB=2654 2024-11-13T16:26:39,907 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1174 is superior to 1024 2024-11-13T16:26:39,923 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:39,925 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:39,926 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:39,928 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21869913, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-21869913, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:39,942 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21869913/hregion-21869913.1731515199929, exclude list is [], retry=0 2024-11-13T16:26:39,945 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:39,946 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:39,947 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:39,949 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21869913/hregion-21869913.1731515199929 2024-11-13T16:26:39,951 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:39,951 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => cb9294c71fc73d47afd40a1759a464d8, NAME => 'testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:39,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741915_1093 (size=61) 2024-11-13T16:26:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741915_1093 (size=61) 2024-11-13T16:26:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741915_1093 (size=61) 2024-11-13T16:26:39,967 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:39,968 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:39,969 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9294c71fc73d47afd40a1759a464d8 columnFamilyName a 2024-11-13T16:26:39,969 DEBUG [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:39,970 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.HStore(327): Store=cb9294c71fc73d47afd40a1759a464d8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:39,970 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:39,971 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:39,971 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:39,972 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:39,972 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:39,973 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:39,975 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:39,976 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb9294c71fc73d47afd40a1759a464d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67236521, jitterRate=0.0019022375345230103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:39,977 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb9294c71fc73d47afd40a1759a464d8: Writing region info on filesystem at 1731515199967Initializing all the Stores at 1731515199968 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515199968Cleaning up temporary data from old regions at 1731515199972 (+4 ms)Region opened successfully at 1731515199977 (+5 ms) 2024-11-13T16:26:39,977 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cb9294c71fc73d47afd40a1759a464d8, disabling compactions & flushes 2024-11-13T16:26:39,977 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:39,977 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:39,977 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. after waiting 0 ms 2024-11-13T16:26:39,977 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:39,977 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:39,977 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cb9294c71fc73d47afd40a1759a464d8: Waiting for close lock at 1731515199977Disabling compacts and flushes for region at 1731515199977Disabling writes for close at 1731515199977Writing region close event to WAL at 1731515199977Closed at 1731515199977 2024-11-13T16:26:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741914_1092 (size=93) 2024-11-13T16:26:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741914_1092 (size=93) 2024-11-13T16:26:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741914_1092 (size=93) 2024-11-13T16:26:39,983 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:39,983 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21869913:(num 1731515199929) 2024-11-13T16:26:39,983 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:39,985 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:40,002 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985, exclude list is [], retry=0 2024-11-13T16:26:40,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:40,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:40,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:40,019 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985 2024-11-13T16:26:40,020 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:40,020 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cb9294c71fc73d47afd40a1759a464d8, NAME => 'testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:40,021 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:40,021 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,021 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,023 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,027 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9294c71fc73d47afd40a1759a464d8 columnFamilyName a 2024-11-13T16:26:40,028 DEBUG [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:40,029 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.HStore(327): Store=cb9294c71fc73d47afd40a1759a464d8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:40,029 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,030 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,035 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,036 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,036 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,043 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,047 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb9294c71fc73d47afd40a1759a464d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66715787, jitterRate=-0.005857303738594055}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:40,048 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb9294c71fc73d47afd40a1759a464d8: Writing region info on filesystem at 1731515200021Initializing all the Stores at 1731515200022 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515200022Cleaning up temporary data from old regions at 1731515200036 (+14 ms)Region opened successfully at 1731515200048 (+12 ms) 2024-11-13T16:26:40,066 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cb9294c71fc73d47afd40a1759a464d8, disabling compactions & flushes 2024-11-13T16:26:40,066 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:40,066 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:40,066 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. after waiting 0 ms 2024-11-13T16:26:40,066 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:40,067 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:40,067 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. 2024-11-13T16:26:40,067 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cb9294c71fc73d47afd40a1759a464d8: Waiting for close lock at 1731515200066Disabling compacts and flushes for region at 1731515200066Disabling writes for close at 1731515200066Writing region close event to WAL at 1731515200067 (+1 ms)Closed at 1731515200067 2024-11-13T16:26:40,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741916_1094 (size=838) 2024-11-13T16:26:40,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741916_1094 (size=838) 2024-11-13T16:26:40,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741916_1094 (size=838) 2024-11-13T16:26:40,096 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985, size=838 (838bytes) 2024-11-13T16:26:40,096 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985 2024-11-13T16:26:40,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985 after 1ms 2024-11-13T16:26:40,100 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:40,101 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985 took 4ms 2024-11-13T16:26:40,107 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985 so closing down 2024-11-13T16:26:40,107 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:40,109 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731515199985.temp 2024-11-13T16:26:40,115 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000003-wal.1731515199985.temp 2024-11-13T16:26:40,115 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:40,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741917_1095 (size=838) 2024-11-13T16:26:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741917_1095 (size=838) 2024-11-13T16:26:40,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741917_1095 (size=838) 2024-11-13T16:26:40,129 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000003-wal.1731515199985.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:40,130 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000003-wal.1731515199985.temp to hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000012 2024-11-13T16:26:40,130 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985, size=838, length=838, corrupted=false, cancelled=false 2024-11-13T16:26:40,131 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985, journal: Splitting hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985, size=838 (838bytes) at 1731515200096Finishing writing output for hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985 so closing down at 1731515200107 (+11 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000003-wal.1731515199985.temp at 1731515200115 (+8 ms)3 split writer threads finished at 1731515200115Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000003-wal.1731515199985.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731515200129 (+14 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000003-wal.1731515199985.temp to hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000012 at 1731515200130 (+1 ms)Processed 10 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985, size=838, length=838, corrupted=false, cancelled=false at 1731515200131 (+1 ms) 2024-11-13T16:26:40,132 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515199985 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515199985 2024-11-13T16:26:40,133 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000012 2024-11-13T16:26:40,138 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:40,669 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:40,672 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:40,688 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515200672, exclude list is [], retry=0 2024-11-13T16:26:40,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:40,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:40,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:40,700 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515200672 2024-11-13T16:26:40,701 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:40,701 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cb9294c71fc73d47afd40a1759a464d8, NAME => 'testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:40,701 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:40,701 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,702 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,706 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,708 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9294c71fc73d47afd40a1759a464d8 columnFamilyName a 2024-11-13T16:26:40,708 DEBUG [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:40,710 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.HStore(327): Store=cb9294c71fc73d47afd40a1759a464d8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:40,710 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,711 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,713 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000012 2024-11-13T16:26:40,717 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:40,720 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000012 2024-11-13T16:26:40,720 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cb9294c71fc73d47afd40a1759a464d8 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T16:26:40,743 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/.tmp/a/b1bea5bba9004b3d9e0edd6d75306469 is 79, key is testDatalossWhenInputError/a:x0/1731515200048/Put/seqid=0 2024-11-13T16:26:40,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741919_1097 (size=5808) 2024-11-13T16:26:40,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741919_1097 (size=5808) 2024-11-13T16:26:40,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741919_1097 (size=5808) 2024-11-13T16:26:40,750 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/.tmp/a/b1bea5bba9004b3d9e0edd6d75306469 2024-11-13T16:26:40,761 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/.tmp/a/b1bea5bba9004b3d9e0edd6d75306469 as hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/a/b1bea5bba9004b3d9e0edd6d75306469 2024-11-13T16:26:40,775 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/a/b1bea5bba9004b3d9e0edd6d75306469, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T16:26:40,776 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for cb9294c71fc73d47afd40a1759a464d8 in 55ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T16:26:40,777 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/0000000000000000012 2024-11-13T16:26:40,778 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,778 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,782 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,784 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T16:26:40,785 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb9294c71fc73d47afd40a1759a464d8; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70966510, jitterRate=0.057483404874801636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:40,786 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb9294c71fc73d47afd40a1759a464d8: Writing region info on filesystem at 1731515200702Initializing all the Stores at 1731515200705 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515200705Obtaining lock to block concurrent updates at 1731515200720 (+15 ms)Preparing flush snapshotting stores in cb9294c71fc73d47afd40a1759a464d8 at 1731515200720Finished memstore snapshotting testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731515200720Flushing stores of testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8. at 1731515200720Flushing cb9294c71fc73d47afd40a1759a464d8/a: creating writer at 1731515200720Flushing cb9294c71fc73d47afd40a1759a464d8/a: appending metadata at 1731515200742 (+22 ms)Flushing cb9294c71fc73d47afd40a1759a464d8/a: closing flushed file at 1731515200742Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41b854e1: reopening flushed file at 1731515200759 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for cb9294c71fc73d47afd40a1759a464d8 in 55ms, sequenceid=12, compaction requested=false; wal=null at 1731515200776 (+17 ms)Cleaning up temporary data from old regions at 1731515200778 (+2 ms)Region opened successfully at 1731515200786 (+8 ms) 2024-11-13T16:26:40,790 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cb9294c71fc73d47afd40a1759a464d8, NAME => 'testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:40,790 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731515199924.cb9294c71fc73d47afd40a1759a464d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:40,790 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,790 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,792 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,793 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb9294c71fc73d47afd40a1759a464d8 columnFamilyName a 2024-11-13T16:26:40,793 DEBUG [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:40,800 DEBUG [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/a/b1bea5bba9004b3d9e0edd6d75306469 2024-11-13T16:26:40,800 INFO [StoreOpener-cb9294c71fc73d47afd40a1759a464d8-1 {}] regionserver.HStore(327): Store=cb9294c71fc73d47afd40a1759a464d8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:40,801 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,801 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,803 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,804 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,806 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cb9294c71fc73d47afd40a1759a464d8 2024-11-13T16:26:40,809 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testDatalossWhenInputError/cb9294c71fc73d47afd40a1759a464d8/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T16:26:40,810 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cb9294c71fc73d47afd40a1759a464d8; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59417272, jitterRate=-0.11461365222930908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T16:26:40,810 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cb9294c71fc73d47afd40a1759a464d8: Writing region info on filesystem at 1731515200790Initializing all the Stores at 1731515200791 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515200791Cleaning up temporary data from old regions at 1731515200804 (+13 ms)Region opened successfully at 1731515200810 (+6 ms) 2024-11-13T16:26:40,831 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=428 (was 418) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37086 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37162 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58838 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-572455086-172.17.0.3-1731515157445:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58824 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58772 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58714 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1256 (was 1174) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=788 (was 788), ProcessCount=11 (was 11), AvailableMemoryMB=2640 (was 2654) 2024-11-13T16:26:40,831 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1256 is superior to 1024 2024-11-13T16:26:40,846 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=428, OpenFileDescriptor=1256, MaxFileDescriptor=1048576, SystemLoadAverage=788, ProcessCount=11, AvailableMemoryMB=2639 2024-11-13T16:26:40,846 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1256 is superior to 1024 2024-11-13T16:26:40,863 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:40,866 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:40,866 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:40,870 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74981390, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-74981390, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:40,888 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74981390/hregion-74981390.1731515200870, exclude list is [], retry=0 2024-11-13T16:26:40,895 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:40,895 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:40,896 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:40,921 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74981390/hregion-74981390.1731515200870 2024-11-13T16:26:40,922 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:40,922 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a4d8f116952b614f5660e2862155d5a0, NAME => 'testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:40,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741921_1099 (size=63) 2024-11-13T16:26:40,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741921_1099 (size=63) 2024-11-13T16:26:40,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741921_1099 (size=63) 2024-11-13T16:26:40,950 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:40,952 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,954 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName a 2024-11-13T16:26:40,954 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:40,955 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:40,955 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,957 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName b 2024-11-13T16:26:40,957 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:40,958 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:40,958 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,961 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName c 2024-11-13T16:26:40,961 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:40,962 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:40,962 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,963 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,970 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,970 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,970 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:40,972 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:40,975 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:40,976 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4d8f116952b614f5660e2862155d5a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64841843, jitterRate=-0.03378124535083771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:40,976 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4d8f116952b614f5660e2862155d5a0: Writing region info on filesystem at 1731515200950Initializing all the Stores at 1731515200951 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515200951Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515200952 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515200952Cleaning up temporary data from old regions at 1731515200970 (+18 ms)Region opened successfully at 1731515200976 (+6 ms) 2024-11-13T16:26:40,977 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4d8f116952b614f5660e2862155d5a0, disabling compactions & flushes 2024-11-13T16:26:40,977 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:40,977 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:40,977 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. after waiting 0 ms 2024-11-13T16:26:40,977 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:40,977 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:40,977 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4d8f116952b614f5660e2862155d5a0: Waiting for close lock at 1731515200976Disabling compacts and flushes for region at 1731515200976Disabling writes for close at 1731515200977 (+1 ms)Writing region close event to WAL at 1731515200977Closed at 1731515200977 2024-11-13T16:26:40,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741920_1098 (size=93) 2024-11-13T16:26:40,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741920_1098 (size=93) 2024-11-13T16:26:40,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741920_1098 (size=93) 2024-11-13T16:26:40,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:40,982 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74981390:(num 1731515200870) 2024-11-13T16:26:40,982 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:40,984 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:40,999 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985, exclude list is [], retry=0 2024-11-13T16:26:41,002 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:41,002 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:41,003 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:41,011 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 2024-11-13T16:26:41,011 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:41,012 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4d8f116952b614f5660e2862155d5a0, NAME => 'testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:41,012 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:41,012 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,012 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,014 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,015 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName a 2024-11-13T16:26:41,015 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:41,016 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:41,016 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,017 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName b 2024-11-13T16:26:41,017 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:41,017 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:41,018 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,019 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName c 2024-11-13T16:26:41,019 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:41,019 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:41,019 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,020 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,022 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,022 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,023 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:41,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:41,025 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4d8f116952b614f5660e2862155d5a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60508553, jitterRate=-0.09835229814052582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:41,026 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4d8f116952b614f5660e2862155d5a0: Writing region info on filesystem at 1731515201012Initializing all the Stores at 1731515201014 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515201014Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515201014Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515201014Cleaning up temporary data from old regions at 1731515201022 (+8 ms)Region opened successfully at 1731515201026 (+4 ms) 2024-11-13T16:26:41,040 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731515201039/Put/seqid=0 2024-11-13T16:26:41,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741923_1101 (size=4875) 2024-11-13T16:26:41,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741923_1101 (size=4875) 2024-11-13T16:26:41,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741923_1101 (size=4875) 2024-11-13T16:26:41,056 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731515201055/Put/seqid=0 2024-11-13T16:26:41,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741924_1102 (size=4875) 2024-11-13T16:26:41,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741924_1102 (size=4875) 2024-11-13T16:26:41,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741924_1102 (size=4875) 2024-11-13T16:26:41,073 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731515201072/Put/seqid=0 2024-11-13T16:26:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741925_1103 (size=4875) 2024-11-13T16:26:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741925_1103 (size=4875) 2024-11-13T16:26:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741925_1103 (size=4875) 2024-11-13T16:26:41,081 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in a4d8f116952b614f5660e2862155d5a0/a 2024-11-13T16:26:41,085 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T16:26:41,086 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T16:26:41,086 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in a4d8f116952b614f5660e2862155d5a0/a 2024-11-13T16:26:41,090 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T16:26:41,090 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T16:26:41,091 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in a4d8f116952b614f5660e2862155d5a0/a 2024-11-13T16:26:41,094 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T16:26:41,094 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T16:26:41,095 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4d8f116952b614f5660e2862155d5a0 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T16:26:41,115 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/.tmp/a/3b90d013ed3949b9a33fd09b76ea6a45 is 55, key is testCompactedBulkLoadedFiles/a:a/1731515201026/Put/seqid=0 2024-11-13T16:26:41,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741926_1104 (size=5107) 2024-11-13T16:26:41,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741926_1104 (size=5107) 2024-11-13T16:26:41,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741926_1104 (size=5107) 2024-11-13T16:26:41,140 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/.tmp/a/3b90d013ed3949b9a33fd09b76ea6a45 2024-11-13T16:26:41,152 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/.tmp/a/3b90d013ed3949b9a33fd09b76ea6a45 as hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3b90d013ed3949b9a33fd09b76ea6a45 2024-11-13T16:26:41,163 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3b90d013ed3949b9a33fd09b76ea6a45, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T16:26:41,167 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for a4d8f116952b614f5660e2862155d5a0 in 72ms, sequenceid=4, compaction requested=false 2024-11-13T16:26:41,167 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a4d8f116952b614f5660e2862155d5a0: 2024-11-13T16:26:41,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_ 2024-11-13T16:26:41,171 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_ 2024-11-13T16:26:41,172 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_ 2024-11-13T16:26:41,173 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile0 into a4d8f116952b614f5660e2862155d5a0/a as hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_ - updating store file list. 2024-11-13T16:26:41,179 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bb6a9ef1693c43739857d38975570606_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T16:26:41,179 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_ into a4d8f116952b614f5660e2862155d5a0/a 2024-11-13T16:26:41,179 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile0 into a4d8f116952b614f5660e2862155d5a0/a (new location: hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_) 2024-11-13T16:26:41,181 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile1 into a4d8f116952b614f5660e2862155d5a0/a as hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_ - updating store file list. 2024-11-13T16:26:41,187 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7645fae707df4335bc8d0311d67f4227_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T16:26:41,187 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_ into a4d8f116952b614f5660e2862155d5a0/a 2024-11-13T16:26:41,187 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile1 into a4d8f116952b614f5660e2862155d5a0/a (new location: hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_) 2024-11-13T16:26:41,189 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile2 into a4d8f116952b614f5660e2862155d5a0/a as hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_ - updating store file list. 2024-11-13T16:26:41,195 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T16:26:41,195 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_ into a4d8f116952b614f5660e2862155d5a0/a 2024-11-13T16:26:41,195 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40955/hbase/testCompactedBulkLoadedFiles/hfile2 into a4d8f116952b614f5660e2862155d5a0/a (new location: hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_) 2024-11-13T16:26:41,204 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T16:26:41,205 DEBUG [Time-limited test {}] regionserver.HStore(1541): a4d8f116952b614f5660e2862155d5a0/a is initiating major compaction (all files) 2024-11-13T16:26:41,205 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of a4d8f116952b614f5660e2862155d5a0/a in testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:41,205 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3b90d013ed3949b9a33fd09b76ea6a45, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_] into tmpdir=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/.tmp, totalSize=19.3 K 2024-11-13T16:26:41,206 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3b90d013ed3949b9a33fd09b76ea6a45, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731515201026 2024-11-13T16:26:41,206 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bb6a9ef1693c43739857d38975570606_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T16:26:41,207 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7645fae707df4335bc8d0311d67f4227_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T16:26:41,208 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T16:26:41,238 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/.tmp/a/aed14310170f4b70962b38679cfb16c1 is 55, key is testCompactedBulkLoadedFiles/a:a/1731515201026/Put/seqid=0 2024-11-13T16:26:41,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741927_1105 (size=6154) 2024-11-13T16:26:41,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741927_1105 (size=6154) 2024-11-13T16:26:41,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741927_1105 (size=6154) 2024-11-13T16:26:41,268 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/.tmp/a/aed14310170f4b70962b38679cfb16c1 as hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/aed14310170f4b70962b38679cfb16c1 2024-11-13T16:26:41,282 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in a4d8f116952b614f5660e2862155d5a0/a of a4d8f116952b614f5660e2862155d5a0 into aed14310170f4b70962b38679cfb16c1(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T16:26:41,282 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for a4d8f116952b614f5660e2862155d5a0: 2024-11-13T16:26:41,282 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T16:26:41,283 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T16:26:41,332 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985, size=0 (0bytes) 2024-11-13T16:26:41,333 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 might be still open, length is 0 2024-11-13T16:26:41,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 2024-11-13T16:26:41,333 WARN [IPC Server handler 4 on default port 40955 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-13T16:26:41,334 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 after 1ms 2024-11-13T16:26:42,685 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T16:26:42,685 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:42,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T16:26:42,686 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T16:26:43,064 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:37210 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:36473:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37210 dst: /127.0.0.1:36473 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36473 remote=/127.0.0.1:37210]. Total timeout mills is 60000, 58216 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:43,064 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58866 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:33543:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58866 dst: /127.0.0.1:33543 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:43,064 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:58884 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:33305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58884 dst: /127.0.0.1:33305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:43,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741922_1106 (size=1368) 2024-11-13T16:26:43,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741922_1106 (size=1368) 2024-11-13T16:26:45,170 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T16:26:45,335 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 after 4002ms 2024-11-13T16:26:45,342 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:45,342 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 took 4010ms 2024-11-13T16:26:45,345 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985; continuing. 2024-11-13T16:26:45,345 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 so closing down 2024-11-13T16:26:45,345 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:45,347 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731515200985.temp 2024-11-13T16:26:45,348 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000003-wal.1731515200985.temp 2024-11-13T16:26:45,349 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:45,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741928_1107 (size=635) 2024-11-13T16:26:45,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741928_1107 (size=635) 2024-11-13T16:26:45,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741928_1107 (size=635) 2024-11-13T16:26:45,357 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000003-wal.1731515200985.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:45,358 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000003-wal.1731515200985.temp to hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000008 2024-11-13T16:26:45,359 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T16:26:45,359 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985, journal: Splitting hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985, size=0 (0bytes) at 1731515201332Finishing writing output for hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 so closing down at 1731515205345 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000003-wal.1731515200985.temp at 1731515205349 (+4 ms)3 split writer threads finished at 1731515205349Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000003-wal.1731515200985.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731515205357 (+8 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000003-wal.1731515200985.temp to hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000008 at 1731515205358 (+1 ms)Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985, size=0, length=0, corrupted=false, cancelled=false at 1731515205359 (+1 ms) 2024-11-13T16:26:45,360 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515200985 2024-11-13T16:26:45,361 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000008 2024-11-13T16:26:45,361 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:45,362 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:45,377 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515205363, exclude list is [], retry=0 2024-11-13T16:26:45,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:45,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:45,380 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:45,381 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515205363 2024-11-13T16:26:45,381 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:45,381 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4d8f116952b614f5660e2862155d5a0, NAME => 'testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:45,382 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:45,382 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,382 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,383 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,384 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName a 2024-11-13T16:26:45,384 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,390 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3b90d013ed3949b9a33fd09b76ea6a45 2024-11-13T16:26:45,393 DEBUG [StoreFileOpener-a4d8f116952b614f5660e2862155d5a0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T16:26:45,394 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_ 2024-11-13T16:26:45,397 DEBUG [StoreFileOpener-a4d8f116952b614f5660e2862155d5a0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7645fae707df4335bc8d0311d67f4227_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T16:26:45,397 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_ 2024-11-13T16:26:45,403 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/aed14310170f4b70962b38679cfb16c1 2024-11-13T16:26:45,407 DEBUG [StoreFileOpener-a4d8f116952b614f5660e2862155d5a0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bb6a9ef1693c43739857d38975570606_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T16:26:45,407 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_ 2024-11-13T16:26:45,407 WARN [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3b90d013ed3949b9a33fd09b76ea6a45 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@719ab9e3 2024-11-13T16:26:45,408 WARN [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@719ab9e3 2024-11-13T16:26:45,408 WARN [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@719ab9e3 2024-11-13T16:26:45,408 WARN [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@719ab9e3 2024-11-13T16:26:45,408 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3b90d013ed3949b9a33fd09b76ea6a45, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_] to archive 2024-11-13T16:26:45,409 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T16:26:45,410 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3b90d013ed3949b9a33fd09b76ea6a45 to hdfs://localhost:40955/hbase/archive/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3b90d013ed3949b9a33fd09b76ea6a45 2024-11-13T16:26:45,411 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_ to hdfs://localhost:40955/hbase/archive/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_ 2024-11-13T16:26:45,412 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_ to hdfs://localhost:40955/hbase/archive/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/7645fae707df4335bc8d0311d67f4227_SeqId_4_ 2024-11-13T16:26:45,413 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_ to hdfs://localhost:40955/hbase/archive/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/a/bb6a9ef1693c43739857d38975570606_SeqId_4_ 2024-11-13T16:26:45,413 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,413 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,414 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName b 2024-11-13T16:26:45,414 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,414 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,414 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,415 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4d8f116952b614f5660e2862155d5a0 columnFamilyName c 2024-11-13T16:26:45,415 DEBUG [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,415 INFO [StoreOpener-a4d8f116952b614f5660e2862155d5a0-1 {}] regionserver.HStore(327): Store=a4d8f116952b614f5660e2862155d5a0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,415 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,417 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,418 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000008 2024-11-13T16:26:45,419 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:45,421 DEBUG [Time-limited test {}] regionserver.HRegion(5836): a4d8f116952b614f5660e2862155d5a0 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "a4d8f116952b614f5660e2862155d5a0" family_name: "a" compaction_input: "3b90d013ed3949b9a33fd09b76ea6a45" compaction_input: "bb6a9ef1693c43739857d38975570606_SeqId_4_" compaction_input: "7645fae707df4335bc8d0311d67f4227_SeqId_4_" compaction_input: "3e9d19a5f2f74f7e9c3d032694cb3bf4_SeqId_4_" compaction_output: "aed14310170f4b70962b38679cfb16c1" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T16:26:45,421 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T16:26:45,421 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000008 2024-11-13T16:26:45,422 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/0000000000000000008 2024-11-13T16:26:45,423 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,423 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,424 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:45,425 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4d8f116952b614f5660e2862155d5a0 2024-11-13T16:26:45,427 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testCompactedBulkLoadedFiles/a4d8f116952b614f5660e2862155d5a0/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T16:26:45,428 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4d8f116952b614f5660e2862155d5a0; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73827755, jitterRate=0.10011927783489227}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:45,428 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4d8f116952b614f5660e2862155d5a0: Writing region info on filesystem at 1731515205382Initializing all the Stores at 1731515205383 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205383Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205383Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205383Cleaning up temporary data from old regions at 1731515205423 (+40 ms)Region opened successfully at 1731515205428 (+5 ms) 2024-11-13T16:26:45,431 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4d8f116952b614f5660e2862155d5a0, disabling compactions & flushes 2024-11-13T16:26:45,431 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:45,431 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:45,431 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. after waiting 0 ms 2024-11-13T16:26:45,432 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:45,432 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731515200864.a4d8f116952b614f5660e2862155d5a0. 2024-11-13T16:26:45,433 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4d8f116952b614f5660e2862155d5a0: Waiting for close lock at 1731515205431Disabling compacts and flushes for region at 1731515205431Disabling writes for close at 1731515205432 (+1 ms)Writing region close event to WAL at 1731515205432Closed at 1731515205432 2024-11-13T16:26:45,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741929_1108 (size=93) 2024-11-13T16:26:45,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741929_1108 (size=93) 2024-11-13T16:26:45,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741929_1108 (size=93) 2024-11-13T16:26:45,438 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:45,438 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731515205363) 2024-11-13T16:26:45,454 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=437 (was 428) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:40955 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_217705015_22 at /127.0.0.1:58960 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_217705015_22 at /127.0.0.1:48142 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40955 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1337 (was 1256) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=757 (was 788), ProcessCount=11 (was 11), AvailableMemoryMB=2648 (was 2639) - AvailableMemoryMB LEAK? - 2024-11-13T16:26:45,454 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1337 is superior to 1024 2024-11-13T16:26:45,470 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=437, OpenFileDescriptor=1337, MaxFileDescriptor=1048576, SystemLoadAverage=757, ProcessCount=11, AvailableMemoryMB=2648 2024-11-13T16:26:45,470 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1337 is superior to 1024 2024-11-13T16:26:45,493 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:45,496 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T16:26:45,497 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T16:26:45,500 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-34034687, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/hregion-34034687, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:45,520 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-34034687/hregion-34034687.1731515205500, exclude list is [], retry=0 2024-11-13T16:26:45,523 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:45,523 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:45,525 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:45,531 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-34034687/hregion-34034687.1731515205500 2024-11-13T16:26:45,532 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125)] 2024-11-13T16:26:45,532 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bf9548951a1d32ac19549aaeb80c0a17, NAME => 'testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40955/hbase 2024-11-13T16:26:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741931_1110 (size=67) 2024-11-13T16:26:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741931_1110 (size=67) 2024-11-13T16:26:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741931_1110 (size=67) 2024-11-13T16:26:45,544 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:45,546 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,547 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName a 2024-11-13T16:26:45,547 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,548 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,548 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,549 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName b 2024-11-13T16:26:45,550 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,550 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,550 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,552 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName c 2024-11-13T16:26:45,552 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,553 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,553 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,559 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,559 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,560 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:45,561 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,564 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T16:26:45,565 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bf9548951a1d32ac19549aaeb80c0a17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72788084, jitterRate=0.08462697267532349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:45,565 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bf9548951a1d32ac19549aaeb80c0a17: Writing region info on filesystem at 1731515205544Initializing all the Stores at 1731515205545 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205545Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205545Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205545Cleaning up temporary data from old regions at 1731515205559 (+14 ms)Region opened successfully at 1731515205565 (+6 ms) 2024-11-13T16:26:45,565 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bf9548951a1d32ac19549aaeb80c0a17, disabling compactions & flushes 2024-11-13T16:26:45,565 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,565 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,566 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. after waiting 0 ms 2024-11-13T16:26:45,566 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,567 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,567 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bf9548951a1d32ac19549aaeb80c0a17: Waiting for close lock at 1731515205565Disabling compacts and flushes for region at 1731515205565Disabling writes for close at 1731515205566 (+1 ms)Writing region close event to WAL at 1731515205567 (+1 ms)Closed at 1731515205567 2024-11-13T16:26:45,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741930_1109 (size=93) 2024-11-13T16:26:45,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741930_1109 (size=93) 2024-11-13T16:26:45,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741930_1109 (size=93) 2024-11-13T16:26:45,577 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:45,577 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-34034687:(num 1731515205500) 2024-11-13T16:26:45,578 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:45,581 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:45,597 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581, exclude list is [], retry=0 2024-11-13T16:26:45,604 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:45,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:45,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:45,644 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581 2024-11-13T16:26:45,647 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565),(127.0.0.1/127.0.0.1:42379:42379)] 2024-11-13T16:26:45,647 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bf9548951a1d32ac19549aaeb80c0a17, NAME => 'testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:45,647 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:45,647 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,647 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,649 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,650 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName a 2024-11-13T16:26:45,650 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,651 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,651 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,652 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName b 2024-11-13T16:26:45,652 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,652 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,652 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,653 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName c 2024-11-13T16:26:45,653 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,654 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,654 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,655 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,656 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,656 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,656 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,657 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:45,658 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,659 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bf9548951a1d32ac19549aaeb80c0a17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62753126, jitterRate=-0.0649055540561676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:45,659 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bf9548951a1d32ac19549aaeb80c0a17: Writing region info on filesystem at 1731515205647Initializing all the Stores at 1731515205649 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205649Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205649Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205649Cleaning up temporary data from old regions at 1731515205656 (+7 ms)Region opened successfully at 1731515205659 (+3 ms) 2024-11-13T16:26:45,669 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bf9548951a1d32ac19549aaeb80c0a17 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T16:26:45,689 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/a/886ceed53fdb42abade9f3d7e07737de is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731515205659/Put/seqid=0 2024-11-13T16:26:45,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741933_1112 (size=5958) 2024-11-13T16:26:45,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741933_1112 (size=5958) 2024-11-13T16:26:45,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741933_1112 (size=5958) 2024-11-13T16:26:45,696 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/a/886ceed53fdb42abade9f3d7e07737de 2024-11-13T16:26:45,702 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/a/886ceed53fdb42abade9f3d7e07737de as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/a/886ceed53fdb42abade9f3d7e07737de 2024-11-13T16:26:45,712 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/a/886ceed53fdb42abade9f3d7e07737de, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T16:26:45,713 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for bf9548951a1d32ac19549aaeb80c0a17 in 44ms, sequenceid=13, compaction requested=false 2024-11-13T16:26:45,713 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bf9548951a1d32ac19549aaeb80c0a17: 2024-11-13T16:26:45,728 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bf9548951a1d32ac19549aaeb80c0a17, disabling compactions & flushes 2024-11-13T16:26:45,728 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,728 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,728 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. after waiting 0 ms 2024-11-13T16:26:45,728 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,729 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,729 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:45,729 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bf9548951a1d32ac19549aaeb80c0a17: Waiting for close lock at 1731515205728Disabling compacts and flushes for region at 1731515205728Disabling writes for close at 1731515205728Writing region close event to WAL at 1731515205729 (+1 ms)Closed at 1731515205729 2024-11-13T16:26:45,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741932_1111 (size=2805) 2024-11-13T16:26:45,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741932_1111 (size=2805) 2024-11-13T16:26:45,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741932_1111 (size=2805) 2024-11-13T16:26:45,748 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581, size=2.7 K (2805bytes) 2024-11-13T16:26:45,748 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581 2024-11-13T16:26:45,748 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581 after 0ms 2024-11-13T16:26:45,751 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:45,751 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581 took 3ms 2024-11-13T16:26:45,753 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581 so closing down 2024-11-13T16:26:45,753 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:45,754 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731515205581.temp 2024-11-13T16:26:45,755 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000003-wal.1731515205581.temp 2024-11-13T16:26:45,755 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:45,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741934_1113 (size=2312) 2024-11-13T16:26:45,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741934_1113 (size=2312) 2024-11-13T16:26:45,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741934_1113 (size=2312) 2024-11-13T16:26:45,762 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000003-wal.1731515205581.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:45,763 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000003-wal.1731515205581.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000035 2024-11-13T16:26:45,763 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-11-13T16:26:45,763 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581, journal: Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581, size=2.7 K (2805bytes) at 1731515205748Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581 so closing down at 1731515205753 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000003-wal.1731515205581.temp at 1731515205755 (+2 ms)3 split writer threads finished at 1731515205755Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000003-wal.1731515205581.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731515205762 (+7 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000003-wal.1731515205581.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000035 at 1731515205763 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1731515205763 2024-11-13T16:26:45,765 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205581 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515205581 2024-11-13T16:26:45,766 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000035 2024-11-13T16:26:45,766 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:45,767 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:45,780 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768, exclude list is [], retry=0 2024-11-13T16:26:45,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:45,783 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:45,783 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:45,785 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 2024-11-13T16:26:45,785 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:45,785 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bf9548951a1d32ac19549aaeb80c0a17, NAME => 'testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17.', STARTKEY => '', ENDKEY => ''} 2024-11-13T16:26:45,785 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:45,785 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,785 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,787 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,788 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName a 2024-11-13T16:26:45,788 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,794 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/a/886ceed53fdb42abade9f3d7e07737de 2024-11-13T16:26:45,794 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,794 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,795 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName b 2024-11-13T16:26:45,795 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,796 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,796 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,797 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName c 2024-11-13T16:26:45,797 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:45,797 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:45,798 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,800 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,801 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000035 2024-11-13T16:26:45,803 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:45,805 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000035 2024-11-13T16:26:45,805 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bf9548951a1d32ac19549aaeb80c0a17 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T16:26:45,822 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/b/7fdac8fec8424855a08cf656470fca68 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731515205713/Put/seqid=0 2024-11-13T16:26:45,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741936_1115 (size=5958) 2024-11-13T16:26:45,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741936_1115 (size=5958) 2024-11-13T16:26:45,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741936_1115 (size=5958) 2024-11-13T16:26:45,831 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/b/7fdac8fec8424855a08cf656470fca68 2024-11-13T16:26:45,854 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/c/fe750926c48b420cb16671bc027ac312 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731515205719/Put/seqid=0 2024-11-13T16:26:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741937_1116 (size=5958) 2024-11-13T16:26:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741937_1116 (size=5958) 2024-11-13T16:26:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741937_1116 (size=5958) 2024-11-13T16:26:45,861 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/c/fe750926c48b420cb16671bc027ac312 2024-11-13T16:26:45,867 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/b/7fdac8fec8424855a08cf656470fca68 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/b/7fdac8fec8424855a08cf656470fca68 2024-11-13T16:26:45,873 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/b/7fdac8fec8424855a08cf656470fca68, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T16:26:45,874 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/c/fe750926c48b420cb16671bc027ac312 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/c/fe750926c48b420cb16671bc027ac312 2024-11-13T16:26:45,879 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/c/fe750926c48b420cb16671bc027ac312, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T16:26:45,879 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for bf9548951a1d32ac19549aaeb80c0a17 in 74ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T16:26:45,880 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000035 2024-11-13T16:26:45,881 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,881 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,882 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:45,883 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:45,886 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T16:26:45,887 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bf9548951a1d32ac19549aaeb80c0a17; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62372217, jitterRate=-0.07058154046535492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:45,887 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bf9548951a1d32ac19549aaeb80c0a17: Writing region info on filesystem at 1731515205785Initializing all the Stores at 1731515205786 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205786Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205787 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515205787Obtaining lock to block concurrent updates at 1731515205805 (+18 ms)Preparing flush snapshotting stores in bf9548951a1d32ac19549aaeb80c0a17 at 1731515205805Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731515205805Flushing stores of testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. at 1731515205805Flushing bf9548951a1d32ac19549aaeb80c0a17/b: creating writer at 1731515205806 (+1 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/b: appending metadata at 1731515205821 (+15 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/b: closing flushed file at 1731515205821Flushing bf9548951a1d32ac19549aaeb80c0a17/c: creating writer at 1731515205838 (+17 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/c: appending metadata at 1731515205853 (+15 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/c: closing flushed file at 1731515205853Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32996dbe: reopening flushed file at 1731515205866 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bc06f94: reopening flushed file at 1731515205873 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for bf9548951a1d32ac19549aaeb80c0a17 in 74ms, sequenceid=35, compaction requested=false; wal=null at 1731515205879 (+6 ms)Cleaning up temporary data from old regions at 1731515205881 (+2 ms)Region opened successfully at 1731515205887 (+6 ms) 2024-11-13T16:26:45,957 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768, size=0 (0bytes) 2024-11-13T16:26:45,957 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 might be still open, length is 0 2024-11-13T16:26:45,957 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 2024-11-13T16:26:45,958 WARN [IPC Server handler 2 on default port 40955 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-13T16:26:45,958 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 after 1ms 2024-11-13T16:26:46,063 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:48204 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:33543:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48204 dst: /127.0.0.1:33543 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33543 remote=/127.0.0.1:48204]. Total timeout mills is 60000, 59851 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:46,063 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:59034 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:36473:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59034 dst: /127.0.0.1:36473 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:46,063 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_520739131_22 at /127.0.0.1:35168 [Receiving block BP-572455086-172.17.0.3-1731515157445:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:33305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35168 dst: /127.0.0.1:33305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:46,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741935_1117 (size=2304) 2024-11-13T16:26:46,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741935_1117 (size=2304) 2024-11-13T16:26:46,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741935_1117 (size=2304) 2024-11-13T16:26:48,169 INFO [master/f8f85b0bb1d8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T16:26:48,169 INFO [master/f8f85b0bb1d8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T16:26:49,959 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 after 4001ms 2024-11-13T16:26:49,962 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:49,962 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 took 4005ms 2024-11-13T16:26:49,973 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768; continuing. 2024-11-13T16:26:49,973 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 so closing down 2024-11-13T16:26:49,973 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T16:26:49,974 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731515205768.temp 2024-11-13T16:26:49,976 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000037-wal.1731515205768.temp 2024-11-13T16:26:49,976 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T16:26:49,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741938_1118 (size=2312) 2024-11-13T16:26:49,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741938_1118 (size=2312) 2024-11-13T16:26:49,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741938_1118 (size=2312) 2024-11-13T16:26:49,984 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000037-wal.1731515205768.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T16:26:49,985 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000037-wal.1731515205768.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000066 2024-11-13T16:26:49,986 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T16:26:49,986 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768, journal: Splitting hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768, size=0 (0bytes) at 1731515205957Finishing writing output for hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 so closing down at 1731515209973 (+4016 ms)Creating recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000037-wal.1731515205768.temp at 1731515209976 (+3 ms)3 split writer threads finished at 1731515209976Closed recovered edits writer path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000037-wal.1731515205768.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731515209984 (+8 ms)Rename recovered edits hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000037-wal.1731515205768.temp to hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000066 at 1731515209986 (+2 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768, size=0, length=0, corrupted=false, cancelled=false at 1731515209986 2024-11-13T16:26:49,988 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 to hdfs://localhost:40955/hbase/oldWALs/wal.1731515205768 2024-11-13T16:26:49,989 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000066 2024-11-13T16:26:49,989 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T16:26:49,991 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40955/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492, archiveDir=hdfs://localhost:40955/hbase/oldWALs, maxLogs=32 2024-11-13T16:26:50,012 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515209992, exclude list is [], retry=0 2024-11-13T16:26:50,015 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33543,DS-5cadb267-b993-475f-b1b9-324c447cbca9,DISK] 2024-11-13T16:26:50,016 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33305,DS-83728dfa-f8a9-4cc2-8f05-31c7c091bbd1,DISK] 2024-11-13T16:26:50,016 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36473,DS-d1d125ef-b087-43f3-b1ac-af30c7d1b6a9,DISK] 2024-11-13T16:26:50,031 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515209992 2024-11-13T16:26:50,032 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42379:42379),(127.0.0.1/127.0.0.1:44125:44125),(127.0.0.1/127.0.0.1:46565:46565)] 2024-11-13T16:26:50,032 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T16:26:50,037 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,040 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName a 2024-11-13T16:26:50,040 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:50,050 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/a/886ceed53fdb42abade9f3d7e07737de 2024-11-13T16:26:50,050 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:50,050 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,051 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName b 2024-11-13T16:26:50,052 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:50,062 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/b/7fdac8fec8424855a08cf656470fca68 2024-11-13T16:26:50,062 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:50,062 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,064 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf9548951a1d32ac19549aaeb80c0a17 columnFamilyName c 2024-11-13T16:26:50,064 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T16:26:50,071 DEBUG [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/c/fe750926c48b420cb16671bc027ac312 2024-11-13T16:26:50,071 INFO [StoreOpener-bf9548951a1d32ac19549aaeb80c0a17-1 {}] regionserver.HStore(327): Store=bf9548951a1d32ac19549aaeb80c0a17/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T16:26:50,071 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,072 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,073 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,074 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000066 2024-11-13T16:26:50,076 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-13T16:26:50,081 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000066 2024-11-13T16:26:50,081 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bf9548951a1d32ac19549aaeb80c0a17 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T16:26:50,102 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/a/a32b292f1b08422e8cea0903dd494042 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731515205894/Put/seqid=0 2024-11-13T16:26:50,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741940_1120 (size=5958) 2024-11-13T16:26:50,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741940_1120 (size=5958) 2024-11-13T16:26:50,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741940_1120 (size=5958) 2024-11-13T16:26:50,114 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/a/a32b292f1b08422e8cea0903dd494042 2024-11-13T16:26:50,139 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/b/0dbfe11e7e944ac8882aa9a12b1fcca6 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731515205901/Put/seqid=0 2024-11-13T16:26:50,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741941_1121 (size=5958) 2024-11-13T16:26:50,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741941_1121 (size=5958) 2024-11-13T16:26:50,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741941_1121 (size=5958) 2024-11-13T16:26:50,551 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/b/0dbfe11e7e944ac8882aa9a12b1fcca6 2024-11-13T16:26:50,581 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/c/ba8f0f3ab8dd42da8da6bbad72822910 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731515205907/Put/seqid=0 2024-11-13T16:26:50,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741942_1122 (size=5958) 2024-11-13T16:26:50,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741942_1122 (size=5958) 2024-11-13T16:26:50,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741942_1122 (size=5958) 2024-11-13T16:26:50,614 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/c/ba8f0f3ab8dd42da8da6bbad72822910 2024-11-13T16:26:50,620 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/a/a32b292f1b08422e8cea0903dd494042 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/a/a32b292f1b08422e8cea0903dd494042 2024-11-13T16:26:50,626 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/a/a32b292f1b08422e8cea0903dd494042, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T16:26:50,627 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/b/0dbfe11e7e944ac8882aa9a12b1fcca6 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/b/0dbfe11e7e944ac8882aa9a12b1fcca6 2024-11-13T16:26:50,632 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/b/0dbfe11e7e944ac8882aa9a12b1fcca6, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T16:26:50,633 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/.tmp/c/ba8f0f3ab8dd42da8da6bbad72822910 as hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/c/ba8f0f3ab8dd42da8da6bbad72822910 2024-11-13T16:26:50,642 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/c/ba8f0f3ab8dd42da8da6bbad72822910, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T16:26:50,642 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for bf9548951a1d32ac19549aaeb80c0a17 in 561ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T16:26:50,643 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/0000000000000000066 2024-11-13T16:26:50,644 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,644 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,645 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T16:26:50,647 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bf9548951a1d32ac19549aaeb80c0a17 2024-11-13T16:26:50,649 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/hbase/data/default/testReplayEditsWrittenViaHRegion/bf9548951a1d32ac19549aaeb80c0a17/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T16:26:50,650 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bf9548951a1d32ac19549aaeb80c0a17; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67307441, jitterRate=0.002959027886390686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T16:26:50,650 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bf9548951a1d32ac19549aaeb80c0a17: Writing region info on filesystem at 1731515210032Initializing all the Stores at 1731515210035 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515210035Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515210037 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731515210037Obtaining lock to block concurrent updates at 1731515210081 (+44 ms)Preparing flush snapshotting stores in bf9548951a1d32ac19549aaeb80c0a17 at 1731515210081Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731515210081Flushing stores of testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. at 1731515210081Flushing bf9548951a1d32ac19549aaeb80c0a17/a: creating writer at 1731515210081Flushing bf9548951a1d32ac19549aaeb80c0a17/a: appending metadata at 1731515210102 (+21 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/a: closing flushed file at 1731515210102Flushing bf9548951a1d32ac19549aaeb80c0a17/b: creating writer at 1731515210120 (+18 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/b: appending metadata at 1731515210139 (+19 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/b: closing flushed file at 1731515210139Flushing bf9548951a1d32ac19549aaeb80c0a17/c: creating writer at 1731515210560 (+421 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/c: appending metadata at 1731515210581 (+21 ms)Flushing bf9548951a1d32ac19549aaeb80c0a17/c: closing flushed file at 1731515210581Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f3f4d7e: reopening flushed file at 1731515210619 (+38 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e9ef2e2: reopening flushed file at 1731515210626 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@108ed21e: reopening flushed file at 1731515210632 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for bf9548951a1d32ac19549aaeb80c0a17 in 561ms, sequenceid=66, compaction requested=false; wal=null at 1731515210642 (+10 ms)Cleaning up temporary data from old regions at 1731515210644 (+2 ms)Region opened successfully at 1731515210650 (+6 ms) 2024-11-13T16:26:50,669 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bf9548951a1d32ac19549aaeb80c0a17, disabling compactions & flushes 2024-11-13T16:26:50,669 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:50,670 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:50,670 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. after waiting 0 ms 2024-11-13T16:26:50,670 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:50,675 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731515205494.bf9548951a1d32ac19549aaeb80c0a17. 2024-11-13T16:26:50,675 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bf9548951a1d32ac19549aaeb80c0a17: Waiting for close lock at 1731515210669Disabling compacts and flushes for region at 1731515210669Disabling writes for close at 1731515210670 (+1 ms)Writing region close event to WAL at 1731515210675 (+5 ms)Closed at 1731515210675 2024-11-13T16:26:50,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741939_1119 (size=93) 2024-11-13T16:26:50,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741939_1119 (size=93) 2024-11-13T16:26:50,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741939_1119 (size=93) 2024-11-13T16:26:50,684 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T16:26:50,685 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731515209992) 2024-11-13T16:26:50,699 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=442 (was 437) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2074418023_22 at /127.0.0.1:59066 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:40955 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2074418023_22 at /127.0.0.1:48246 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2074418023_22 at /127.0.0.1:35218 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:40955 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1400 (was 1337) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=737 (was 757), ProcessCount=11 (was 11), AvailableMemoryMB=3330 (was 2648) - AvailableMemoryMB LEAK? - 2024-11-13T16:26:50,700 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1400 is superior to 1024 2024-11-13T16:26:50,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T16:26:50,700 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T16:26:50,700 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:26:50,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:50,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:50,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T16:26:50,701 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T16:26:50,702 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=387325396, stopped=false 2024-11-13T16:26:50,702 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f8f85b0bb1d8,34015,1731515162192 2024-11-13T16:26:50,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T16:26:50,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T16:26:50,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T16:26:50,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:50,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:50,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:50,735 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:26:50,736 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T16:26:50,736 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:26:50,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:50,736 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f8f85b0bb1d8,37847,1731515163204' ***** 2024-11-13T16:26:50,737 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T16:26:50,737 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f8f85b0bb1d8,40565,1731515163497' ***** 2024-11-13T16:26:50,737 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T16:26:50,737 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T16:26:50,737 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T16:26:50,737 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:26:50,737 INFO [RS:0;f8f85b0bb1d8:37847 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T16:26:50,737 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T16:26:50,737 INFO [RS:2;f8f85b0bb1d8:40565 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T16:26:50,737 INFO [RS:0;f8f85b0bb1d8:37847 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T16:26:50,737 INFO [RS:2;f8f85b0bb1d8:40565 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T16:26:50,737 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:26:50,737 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(3091): Received CLOSE for 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:50,737 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(959): stopping server f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:50,737 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(959): stopping server f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:50,737 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T16:26:50,737 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T16:26:50,737 INFO [RS:0;f8f85b0bb1d8:37847 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f8f85b0bb1d8:37847. 2024-11-13T16:26:50,738 INFO [RS:2;f8f85b0bb1d8:40565 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f8f85b0bb1d8:40565. 2024-11-13T16:26:50,738 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:26:50,738 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:50,738 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 32b359af0da6a75b36c596a655fd4546, disabling compactions & flushes 2024-11-13T16:26:50,738 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T16:26:50,738 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:50,738 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T16:26:50,738 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:50,738 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:50,738 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1325): Online Regions={32b359af0da6a75b36c596a655fd4546=testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546.} 2024-11-13T16:26:50,738 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. after waiting 0 ms 2024-11-13T16:26:50,738 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:50,738 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1351): Waiting on 32b359af0da6a75b36c596a655fd4546 2024-11-13T16:26:50,738 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T16:26:50,738 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T16:26:50,738 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T16:26:50,738 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T16:26:50,738 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T16:26:50,738 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T16:26:50,739 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T16:26:50,739 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T16:26:50,739 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T16:26:50,739 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T16:26:50,739 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T16:26:50,739 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T16:26:50,739 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T16:26:50,739 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T16:26:50,740 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-13T16:26:50,745 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/default/testReplayEditsAfterRegionMovedWithMultiCF/32b359af0da6a75b36c596a655fd4546/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T16:26:50,745 INFO [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:50,746 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 32b359af0da6a75b36c596a655fd4546: Waiting for close lock at 1731515210737Running coprocessor pre-close hooks at 1731515210737Disabling compacts and flushes for region at 1731515210737Disabling writes for close at 1731515210738 (+1 ms)Writing region close event to WAL at 1731515210740 (+2 ms)Running coprocessor post-close hooks at 1731515210745 (+5 ms)Closed at 1731515210745 2024-11-13T16:26:50,746 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/info/959479ca72054b78a3f1db2d932cdccc is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546./info:regioninfo/1731515189063/Put/seqid=0 2024-11-13T16:26:50,746 DEBUG [RS_CLOSE_REGION-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731515183621.32b359af0da6a75b36c596a655fd4546. 2024-11-13T16:26:50,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741943_1123 (size=6778) 2024-11-13T16:26:50,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741943_1123 (size=6778) 2024-11-13T16:26:50,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741943_1123 (size=6778) 2024-11-13T16:26:50,756 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/info/959479ca72054b78a3f1db2d932cdccc 2024-11-13T16:26:50,767 INFO [regionserver/f8f85b0bb1d8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:26:50,770 INFO [regionserver/f8f85b0bb1d8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:26:50,775 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/.tmp/info/959479ca72054b78a3f1db2d932cdccc as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/info/959479ca72054b78a3f1db2d932cdccc 2024-11-13T16:26:50,789 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/info/959479ca72054b78a3f1db2d932cdccc, entries=8, sequenceid=23, filesize=6.6 K 2024-11-13T16:26:50,790 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 51ms, sequenceid=23, compaction requested=false 2024-11-13T16:26:50,803 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-13T16:26:50,804 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T16:26:50,804 INFO [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T16:26:50,804 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731515210739Running coprocessor pre-close hooks at 1731515210739Disabling compacts and flushes for region at 1731515210739Disabling writes for close at 1731515210739Obtaining lock to block concurrent updates at 1731515210740 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731515210740Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731515210740Flushing stores of hbase:meta,,1.1588230740 at 1731515210741 (+1 ms)Flushing 1588230740/info: creating writer at 1731515210741Flushing 1588230740/info: appending metadata at 1731515210745 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731515210745Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a04f018: reopening flushed file at 1731515210770 (+25 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 51ms, sequenceid=23, compaction requested=false at 1731515210790 (+20 ms)Writing region close event to WAL at 1731515210794 (+4 ms)Running coprocessor post-close hooks at 1731515210804 (+10 ms)Closed at 1731515210804 2024-11-13T16:26:50,805 DEBUG [RS_CLOSE_META-regionserver/f8f85b0bb1d8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T16:26:50,831 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T16:26:50,831 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T16:26:50,858 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T16:26:50,858 INFO [regionserver/f8f85b0bb1d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T16:26:50,938 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(976): stopping server f8f85b0bb1d8,37847,1731515163204; all regions closed. 2024-11-13T16:26:50,939 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(976): stopping server f8f85b0bb1d8,40565,1731515163497; all regions closed. 2024-11-13T16:26:50,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741835_1011 (size=2652) 2024-11-13T16:26:50,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741835_1011 (size=2652) 2024-11-13T16:26:50,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741893_1071 (size=1552) 2024-11-13T16:26:50,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741893_1071 (size=1552) 2024-11-13T16:26:50,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741893_1071 (size=1552) 2024-11-13T16:26:50,944 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs 2024-11-13T16:26:50,945 INFO [RS:0;f8f85b0bb1d8:37847 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f8f85b0bb1d8%2C37847%2C1731515163204:(num 1731515165968) 2024-11-13T16:26:50,945 DEBUG [RS:0;f8f85b0bb1d8:37847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:50,945 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:26:50,945 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T16:26:50,945 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.ChoreService(370): Chore service for: regionserver/f8f85b0bb1d8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T16:26:50,945 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T16:26:50,945 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T16:26:50,946 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T16:26:50,946 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T16:26:50,946 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs 2024-11-13T16:26:50,946 INFO [RS:0;f8f85b0bb1d8:37847 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37847 2024-11-13T16:26:50,946 INFO [RS:2;f8f85b0bb1d8:40565 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f8f85b0bb1d8%2C40565%2C1731515163497.meta:.meta(num 1731515187722) 2024-11-13T16:26:50,946 INFO [regionserver/f8f85b0bb1d8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T16:26:50,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741833_1009 (size=93) 2024-11-13T16:26:50,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741833_1009 (size=93) 2024-11-13T16:26:50,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T16:26:50,957 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f8f85b0bb1d8,37847,1731515163204] 2024-11-13T16:26:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741833_1009 (size=93) 2024-11-13T16:26:50,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f8f85b0bb1d8,37847,1731515163204 2024-11-13T16:26:50,957 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T16:26:50,961 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/oldWALs 2024-11-13T16:26:50,961 INFO [RS:2;f8f85b0bb1d8:40565 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f8f85b0bb1d8%2C40565%2C1731515163497:(num 1731515165964) 2024-11-13T16:26:50,961 DEBUG [RS:2;f8f85b0bb1d8:40565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T16:26:50,961 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T16:26:50,961 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T16:26:50,961 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.ChoreService(370): Chore service for: regionserver/f8f85b0bb1d8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T16:26:50,962 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T16:26:50,962 INFO [RS:2;f8f85b0bb1d8:40565 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40565 2024-11-13T16:26:50,962 INFO [regionserver/f8f85b0bb1d8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T16:26:50,963 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f8f85b0bb1d8,37847,1731515163204 already deleted, retry=false 2024-11-13T16:26:50,964 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f8f85b0bb1d8,37847,1731515163204 expired; onlineServers=1 2024-11-13T16:26:50,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f8f85b0bb1d8,40565,1731515163497 2024-11-13T16:26:50,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T16:26:50,973 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T16:26:50,973 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f8f85b0bb1d8,40565,1731515163497] 2024-11-13T16:26:50,989 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f8f85b0bb1d8,40565,1731515163497 already deleted, retry=false 2024-11-13T16:26:50,989 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f8f85b0bb1d8,40565,1731515163497 expired; onlineServers=0 2024-11-13T16:26:50,989 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f8f85b0bb1d8,34015,1731515162192' ***** 2024-11-13T16:26:50,989 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T16:26:50,989 INFO [M:0;f8f85b0bb1d8:34015 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T16:26:50,989 INFO [M:0;f8f85b0bb1d8:34015 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T16:26:50,989 DEBUG [M:0;f8f85b0bb1d8:34015 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T16:26:50,990 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T16:26:50,990 DEBUG [M:0;f8f85b0bb1d8:34015 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T16:26:50,990 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.large.0-1731515165425 {}] cleaner.HFileCleaner(306): Exit Thread[master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.large.0-1731515165425,5,FailOnTimeoutGroup] 2024-11-13T16:26:50,990 DEBUG [master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.small.0-1731515165428 {}] cleaner.HFileCleaner(306): Exit Thread[master/f8f85b0bb1d8:0:becomeActiveMaster-HFileCleaner.small.0-1731515165428,5,FailOnTimeoutGroup] 2024-11-13T16:26:50,990 INFO [M:0;f8f85b0bb1d8:34015 {}] hbase.ChoreService(370): Chore service for: master/f8f85b0bb1d8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T16:26:50,990 INFO [M:0;f8f85b0bb1d8:34015 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T16:26:50,990 DEBUG [M:0;f8f85b0bb1d8:34015 {}] master.HMaster(1795): Stopping service threads 2024-11-13T16:26:50,990 INFO [M:0;f8f85b0bb1d8:34015 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T16:26:50,990 INFO [M:0;f8f85b0bb1d8:34015 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T16:26:50,991 INFO [M:0;f8f85b0bb1d8:34015 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T16:26:50,991 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T16:26:51,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T16:26:51,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T16:26:51,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:26:51,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37847-0x101363260180001, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:26:51,072 DEBUG [M:0;f8f85b0bb1d8:34015 {}] zookeeper.ZKUtil(347): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T16:26:51,072 INFO [RS:0;f8f85b0bb1d8:37847 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T16:26:51,072 INFO [RS:0;f8f85b0bb1d8:37847 {}] regionserver.HRegionServer(1031): Exiting; stopping=f8f85b0bb1d8,37847,1731515163204; zookeeper connection closed. 2024-11-13T16:26:51,072 WARN [M:0;f8f85b0bb1d8:34015 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T16:26:51,073 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@46c9db47 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@46c9db47 2024-11-13T16:26:51,073 INFO [M:0;f8f85b0bb1d8:34015 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/.lastflushedseqids 2024-11-13T16:26:51,080 INFO [RS:2;f8f85b0bb1d8:40565 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T16:26:51,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:26:51,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40565-0x101363260180003, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:26:51,081 INFO [RS:2;f8f85b0bb1d8:40565 {}] regionserver.HRegionServer(1031): Exiting; stopping=f8f85b0bb1d8,40565,1731515163497; zookeeper connection closed. 2024-11-13T16:26:51,081 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@62c54f5a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@62c54f5a 2024-11-13T16:26:51,081 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T16:26:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741944_1124 (size=119) 2024-11-13T16:26:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741944_1124 (size=119) 2024-11-13T16:26:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741944_1124 (size=119) 2024-11-13T16:26:51,089 INFO [M:0;f8f85b0bb1d8:34015 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T16:26:51,089 INFO [M:0;f8f85b0bb1d8:34015 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T16:26:51,089 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T16:26:51,089 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:26:51,089 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:26:51,089 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T16:26:51,089 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:26:51,089 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-11-13T16:26:51,105 DEBUG [M:0;f8f85b0bb1d8:34015 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3c2db9732fca4098be8e722afe06ae97 is 82, key is hbase:meta,,1/info:regioninfo/1731515188379/Put/seqid=0 2024-11-13T16:26:51,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741945_1125 (size=6063) 2024-11-13T16:26:51,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741945_1125 (size=6063) 2024-11-13T16:26:51,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741945_1125 (size=6063) 2024-11-13T16:26:51,112 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3c2db9732fca4098be8e722afe06ae97 2024-11-13T16:26:51,133 DEBUG [M:0;f8f85b0bb1d8:34015 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15092207b53d4e65943bbce65fcdea16 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731515184136/Put/seqid=0 2024-11-13T16:26:51,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741946_1126 (size=7908) 2024-11-13T16:26:51,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741946_1126 (size=7908) 2024-11-13T16:26:51,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741946_1126 (size=7908) 2024-11-13T16:26:51,144 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15092207b53d4e65943bbce65fcdea16 2024-11-13T16:26:51,148 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 15092207b53d4e65943bbce65fcdea16 2024-11-13T16:26:51,163 DEBUG [M:0;f8f85b0bb1d8:34015 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2277251b94694fdda38efa501e19c35a is 69, key is f8f85b0bb1d8,37847,1731515163204/rs:state/1731515165602/Put/seqid=0 2024-11-13T16:26:51,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741947_1127 (size=5440) 2024-11-13T16:26:51,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36473 is added to blk_1073741947_1127 (size=5440) 2024-11-13T16:26:51,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741947_1127 (size=5440) 2024-11-13T16:26:51,178 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2277251b94694fdda38efa501e19c35a 2024-11-13T16:26:51,184 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2277251b94694fdda38efa501e19c35a 2024-11-13T16:26:51,185 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3c2db9732fca4098be8e722afe06ae97 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3c2db9732fca4098be8e722afe06ae97 2024-11-13T16:26:51,190 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3c2db9732fca4098be8e722afe06ae97, entries=14, sequenceid=207, filesize=5.9 K 2024-11-13T16:26:51,192 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15092207b53d4e65943bbce65fcdea16 as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/15092207b53d4e65943bbce65fcdea16 2024-11-13T16:26:51,198 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 15092207b53d4e65943bbce65fcdea16 2024-11-13T16:26:51,199 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/15092207b53d4e65943bbce65fcdea16, entries=21, sequenceid=207, filesize=7.7 K 2024-11-13T16:26:51,200 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2277251b94694fdda38efa501e19c35a as hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2277251b94694fdda38efa501e19c35a 2024-11-13T16:26:51,209 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2277251b94694fdda38efa501e19c35a 2024-11-13T16:26:51,209 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40955/user/jenkins/test-data/4335ef82-2c87-df4c-2cbd-c343f102af19/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2277251b94694fdda38efa501e19c35a, entries=3, sequenceid=207, filesize=5.3 K 2024-11-13T16:26:51,211 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 122ms, sequenceid=207, compaction requested=false 2024-11-13T16:26:51,213 INFO [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T16:26:51,213 DEBUG [M:0;f8f85b0bb1d8:34015 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731515211089Disabling compacts and flushes for region at 1731515211089Disabling writes for close at 1731515211089Obtaining lock to block concurrent updates at 1731515211089Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731515211089Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1731515211090 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731515211090Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731515211090Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731515211105 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731515211105Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731515211117 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731515211132 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731515211133 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731515211149 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731515211163 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731515211163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75cc3ac8: reopening flushed file at 1731515211184 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5162061a: reopening flushed file at 1731515211191 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74cac83f: reopening flushed file at 1731515211199 (+8 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 122ms, sequenceid=207, compaction requested=false at 1731515211211 (+12 ms)Writing region close event to WAL at 1731515211213 (+2 ms)Closed at 1731515211213 2024-11-13T16:26:51,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33543 is added to blk_1073741830_1006 (size=86759) 2024-11-13T16:26:51,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33305 is added to blk_1073741830_1006 (size=86759) 2024-11-13T16:26:51,217 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T16:26:51,217 INFO [M:0;f8f85b0bb1d8:34015 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T16:26:51,217 INFO [M:0;f8f85b0bb1d8:34015 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34015 2024-11-13T16:26:51,217 INFO [M:0;f8f85b0bb1d8:34015 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T16:26:51,457 INFO [M:0;f8f85b0bb1d8:34015 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T16:26:51,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:26:51,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34015-0x101363260180000, quorum=127.0.0.1:54878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T16:26:51,466 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515199693 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515199693 (inode 16688) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731515190376/wal.1731515199693 (inode 16688) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T16:26:51,470 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515190188 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515190188 (inode 16665) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731515189360/wal.1731515190188 (inode 16665) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T16:26:51,471 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731515178889/wal.1731515179152 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:51,474 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515200672 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515200672 (inode 16714) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731515199923/wal.1731515200672 (inode 16714) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T16:26:51,475 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731515200862/wal.1731515200985 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:51,475 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731515205492/wal.1731515205768 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:51,476 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731515170616/wal.1731515170830 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T16:26:51,480 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal.1731515170297 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal.1731515170297 (inode 16485) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731515169333/wal.1731515170297 (inode 16485) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T16:26:51,488 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal.1731515169042 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal.1731515169042 (inode 16462) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731515167596/wal.1731515169042 (inode 16462) Holder DFSClient_NONMAPREDUCE_520739131_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T16:26:51,499 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:26:51,502 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T16:26:51,502 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T16:26:51,502 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T16:26:51,502 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir/,STOPPED} 2024-11-13T16:26:51,513 WARN [BP-572455086-172.17.0.3-1731515157445 heartbeating to localhost/127.0.0.1:40955 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T16:26:51,513 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T16:26:51,513 WARN [BP-572455086-172.17.0.3-1731515157445 heartbeating to localhost/127.0.0.1:40955 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-572455086-172.17.0.3-1731515157445 (Datanode Uuid 7942aa5b-900b-4db3-b69a-89950f743638) service to localhost/127.0.0.1:40955 2024-11-13T16:26:51,513 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T16:26:51,515 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data5/current/BP-572455086-172.17.0.3-1731515157445 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:26:51,516 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T16:26:51,516 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data6/current/BP-572455086-172.17.0.3-1731515157445 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:26:51,524 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:26:51,525 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T16:26:51,525 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T16:26:51,525 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T16:26:51,525 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir/,STOPPED} 2024-11-13T16:26:51,527 WARN [BP-572455086-172.17.0.3-1731515157445 heartbeating to localhost/127.0.0.1:40955 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T16:26:51,527 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T16:26:51,527 WARN [BP-572455086-172.17.0.3-1731515157445 heartbeating to localhost/127.0.0.1:40955 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-572455086-172.17.0.3-1731515157445 (Datanode Uuid 2db7fc52-8d40-4da7-b13b-25d17df12739) service to localhost/127.0.0.1:40955 2024-11-13T16:26:51,527 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T16:26:51,528 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data3/current/BP-572455086-172.17.0.3-1731515157445 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:26:51,528 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data4/current/BP-572455086-172.17.0.3-1731515157445 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:26:51,529 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T16:26:51,537 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T16:26:51,537 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T16:26:51,538 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T16:26:51,538 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T16:26:51,538 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir/,STOPPED} 2024-11-13T16:26:51,541 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T16:26:51,541 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T16:26:51,542 WARN [BP-572455086-172.17.0.3-1731515157445 heartbeating to localhost/127.0.0.1:40955 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T16:26:51,542 WARN [BP-572455086-172.17.0.3-1731515157445 heartbeating to localhost/127.0.0.1:40955 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-572455086-172.17.0.3-1731515157445 (Datanode Uuid 7adbdbbb-b812-4222-a2c5-0af984f390b0) service to localhost/127.0.0.1:40955 2024-11-13T16:26:51,544 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data2/current/BP-572455086-172.17.0.3-1731515157445 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:26:51,544 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T16:26:51,544 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/cluster_67aa0f4b-eb52-e4d4-e478-d83e993f7066/data/data1/current/BP-572455086-172.17.0.3-1731515157445 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T16:26:51,559 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T16:26:51,566 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T16:26:51,566 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T16:26:51,567 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T16:26:51,567 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/77a3c496-f7f4-300e-4dbc-950fa2f65208/hadoop.log.dir/,STOPPED} 2024-11-13T16:26:51,585 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T16:26:51,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down