2024-11-08 12:44:16,382 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba 2024-11-08 12:44:16,399 main DEBUG Took 0.014372 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 12:44:16,400 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 12:44:16,400 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 12:44:16,402 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 12:44:16,403 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,416 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 12:44:16,478 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,480 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,481 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,482 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,482 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,483 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,484 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,484 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,485 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,485 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,487 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,488 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,489 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,490 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,491 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,492 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,493 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 12:44:16,494 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,495 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 12:44:16,497 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 12:44:16,499 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 12:44:16,501 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 12:44:16,502 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 12:44:16,503 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 12:44:16,504 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 12:44:16,516 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 12:44:16,519 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 12:44:16,521 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 12:44:16,522 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 12:44:16,522 main DEBUG createAppenders(={Console}) 2024-11-08 12:44:16,523 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba initialized 2024-11-08 12:44:16,524 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba 2024-11-08 12:44:16,524 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@24569dba OK. 2024-11-08 12:44:16,525 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 12:44:16,526 main DEBUG OutputStream closed 2024-11-08 12:44:16,526 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 12:44:16,526 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 12:44:16,527 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@49c7b90e OK 2024-11-08 12:44:16,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 12:44:16,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 12:44:16,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 12:44:16,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 12:44:16,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 12:44:16,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 12:44:16,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 12:44:16,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 12:44:16,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 12:44:16,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 12:44:16,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 12:44:16,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 12:44:16,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 12:44:16,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 12:44:16,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 12:44:16,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 12:44:16,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 12:44:16,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 12:44:16,635 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 12:44:16,635 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@35432107) with optional ClassLoader: null 2024-11-08 12:44:16,636 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 12:44:16,636 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@35432107] started OK. 2024-11-08T12:44:16,657 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC timeout: 26 mins 2024-11-08 12:44:16,661 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 12:44:16,661 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T12:44:17,062 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4 2024-11-08T12:44:17,112 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5, deleteOnExit=true 2024-11-08T12:44:17,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/test.cache.data in system properties and HBase conf 2024-11-08T12:44:17,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T12:44:17,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir in system properties and HBase conf 2024-11-08T12:44:17,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T12:44:17,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T12:44:17,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T12:44:17,285 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T12:44:17,416 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T12:44:17,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T12:44:17,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T12:44:17,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T12:44:17,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T12:44:17,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T12:44:17,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T12:44:17,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T12:44:17,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T12:44:17,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T12:44:17,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/nfs.dump.dir in system properties and HBase conf 2024-11-08T12:44:17,433 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/java.io.tmpdir in system properties and HBase conf 2024-11-08T12:44:17,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T12:44:17,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T12:44:17,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T12:44:18,684 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T12:44:18,810 INFO [Time-limited test {}] log.Log(170): Logging initialized @3508ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T12:44:18,928 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:19,031 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T12:44:19,079 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T12:44:19,085 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T12:44:19,087 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T12:44:19,125 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:19,131 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e18bd18{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir/,AVAILABLE} 2024-11-08T12:44:19,132 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2faf2775{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T12:44:19,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@76e22261{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/java.io.tmpdir/jetty-localhost-36447-hadoop-hdfs-3_4_1-tests_jar-_-any-8774038706505044177/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T12:44:19,513 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3599471c{HTTP/1.1, (http/1.1)}{localhost:36447} 2024-11-08T12:44:19,514 INFO [Time-limited test {}] server.Server(415): Started @4213ms 2024-11-08T12:44:20,131 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:20,151 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T12:44:20,166 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T12:44:20,166 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T12:44:20,167 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T12:44:20,168 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51c14b1c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir/,AVAILABLE} 2024-11-08T12:44:20,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5f7daeb1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T12:44:20,346 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@321ada2c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/java.io.tmpdir/jetty-localhost-45525-hadoop-hdfs-3_4_1-tests_jar-_-any-2848924308452452339/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:20,347 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@637479e9{HTTP/1.1, (http/1.1)}{localhost:45525} 2024-11-08T12:44:20,348 INFO [Time-limited test {}] server.Server(415): Started @5047ms 2024-11-08T12:44:20,419 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T12:44:20,648 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:20,658 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T12:44:20,686 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T12:44:20,686 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T12:44:20,686 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T12:44:20,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10ea181c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir/,AVAILABLE} 2024-11-08T12:44:20,697 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1663c233{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T12:44:20,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3ea31b9a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/java.io.tmpdir/jetty-localhost-41367-hadoop-hdfs-3_4_1-tests_jar-_-any-7180314037518412209/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:20,877 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@67374f99{HTTP/1.1, (http/1.1)}{localhost:41367} 2024-11-08T12:44:20,877 INFO [Time-limited test {}] server.Server(415): Started @5576ms 2024-11-08T12:44:20,881 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T12:44:20,959 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:20,966 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T12:44:21,001 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T12:44:21,005 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T12:44:21,005 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T12:44:21,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@513d748e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir/,AVAILABLE} 2024-11-08T12:44:21,015 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7353365d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T12:44:21,094 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data1/current/BP-1671604421-172.17.0.3-1731069858349/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:21,094 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data4/current/BP-1671604421-172.17.0.3-1731069858349/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:21,095 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data3/current/BP-1671604421-172.17.0.3-1731069858349/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:21,100 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data2/current/BP-1671604421-172.17.0.3-1731069858349/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:21,206 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54572a6d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/java.io.tmpdir/jetty-localhost-43489-hadoop-hdfs-3_4_1-tests_jar-_-any-8103443822061124073/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:21,209 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54a3c380{HTTP/1.1, (http/1.1)}{localhost:43489} 2024-11-08T12:44:21,209 INFO [Time-limited test {}] server.Server(415): Started @5908ms 2024-11-08T12:44:21,212 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T12:44:21,221 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T12:44:21,222 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T12:44:21,317 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x15f83c401e943749 with lease ID 0x4134bdfff1b89764: Processing first storage report for DS-945a1a34-020c-44eb-9686-eccd68d5556b from datanode DatanodeRegistration(127.0.0.1:39559, datanodeUuid=2a614efd-e11c-454c-9df3-621e978018cc, infoPort=36629, infoSecurePort=0, ipcPort=46533, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349) 2024-11-08T12:44:21,319 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x15f83c401e943749 with lease ID 0x4134bdfff1b89764: from storage DS-945a1a34-020c-44eb-9686-eccd68d5556b node DatanodeRegistration(127.0.0.1:39559, datanodeUuid=2a614efd-e11c-454c-9df3-621e978018cc, infoPort=36629, infoSecurePort=0, ipcPort=46533, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T12:44:21,319 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0573c60a76dd568 with lease ID 0x4134bdfff1b89765: Processing first storage report for DS-b813e00c-15be-4555-9ef3-4c07ec8f3d51 from datanode DatanodeRegistration(127.0.0.1:40469, datanodeUuid=ca8f41a9-2645-4d4d-87ff-eaeee3311fc5, infoPort=41221, infoSecurePort=0, ipcPort=42653, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349) 2024-11-08T12:44:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0573c60a76dd568 with lease ID 0x4134bdfff1b89765: from storage DS-b813e00c-15be-4555-9ef3-4c07ec8f3d51 node DatanodeRegistration(127.0.0.1:40469, datanodeUuid=ca8f41a9-2645-4d4d-87ff-eaeee3311fc5, infoPort=41221, infoSecurePort=0, ipcPort=42653, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T12:44:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x15f83c401e943749 with lease ID 0x4134bdfff1b89764: Processing first storage report for DS-f894c81b-5f87-4f6c-a270-1fcea80bbabc from datanode DatanodeRegistration(127.0.0.1:39559, datanodeUuid=2a614efd-e11c-454c-9df3-621e978018cc, infoPort=36629, infoSecurePort=0, ipcPort=46533, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349) 2024-11-08T12:44:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x15f83c401e943749 with lease ID 0x4134bdfff1b89764: from storage DS-f894c81b-5f87-4f6c-a270-1fcea80bbabc node DatanodeRegistration(127.0.0.1:39559, datanodeUuid=2a614efd-e11c-454c-9df3-621e978018cc, infoPort=36629, infoSecurePort=0, ipcPort=46533, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T12:44:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0573c60a76dd568 with lease ID 0x4134bdfff1b89765: Processing first storage report for DS-82a63566-2316-42f7-a2db-f164efcceafd from datanode DatanodeRegistration(127.0.0.1:40469, datanodeUuid=ca8f41a9-2645-4d4d-87ff-eaeee3311fc5, infoPort=41221, infoSecurePort=0, ipcPort=42653, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349) 2024-11-08T12:44:21,321 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0573c60a76dd568 with lease ID 0x4134bdfff1b89765: from storage DS-82a63566-2316-42f7-a2db-f164efcceafd node DatanodeRegistration(127.0.0.1:40469, datanodeUuid=ca8f41a9-2645-4d4d-87ff-eaeee3311fc5, infoPort=41221, infoSecurePort=0, ipcPort=42653, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T12:44:21,489 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data5/current/BP-1671604421-172.17.0.3-1731069858349/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:21,510 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data6/current/BP-1671604421-172.17.0.3-1731069858349/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:21,584 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T12:44:21,593 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4529a8bb13c2701f with lease ID 0x4134bdfff1b89766: Processing first storage report for DS-090cfee0-ba02-4ad2-87d7-fd886a424c14 from datanode DatanodeRegistration(127.0.0.1:34381, datanodeUuid=7f9f972a-bd45-47f8-9d9b-852f924a86b9, infoPort=36231, infoSecurePort=0, ipcPort=37975, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349) 2024-11-08T12:44:21,593 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4529a8bb13c2701f with lease ID 0x4134bdfff1b89766: from storage DS-090cfee0-ba02-4ad2-87d7-fd886a424c14 node DatanodeRegistration(127.0.0.1:34381, datanodeUuid=7f9f972a-bd45-47f8-9d9b-852f924a86b9, infoPort=36231, infoSecurePort=0, ipcPort=37975, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T12:44:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4529a8bb13c2701f with lease ID 0x4134bdfff1b89766: Processing first storage report for DS-5e6eb891-e8e1-41c0-b81e-00298cd74980 from datanode DatanodeRegistration(127.0.0.1:34381, datanodeUuid=7f9f972a-bd45-47f8-9d9b-852f924a86b9, infoPort=36231, infoSecurePort=0, ipcPort=37975, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349) 2024-11-08T12:44:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4529a8bb13c2701f with lease ID 0x4134bdfff1b89766: from storage DS-5e6eb891-e8e1-41c0-b81e-00298cd74980 node DatanodeRegistration(127.0.0.1:34381, datanodeUuid=7f9f972a-bd45-47f8-9d9b-852f924a86b9, infoPort=36231, infoSecurePort=0, ipcPort=37975, storageInfo=lv=-57;cid=testClusterID;nsid=339547121;c=1731069858349), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T12:44:21,834 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4 2024-11-08T12:44:21,960 WARN [Time-limited test {}] erasurecode.ErasureCodeNative(55): ISA-L support is not available in your platform... using builtin-java codec where applicable 2024-11-08T12:44:22,050 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestHBaseWalOnEC#testReadWrite[0] Thread=157, OpenFileDescriptor=391, MaxFileDescriptor=1048576, SystemLoadAverage=379, ProcessCount=11, AvailableMemoryMB=2747 2024-11-08T12:44:22,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T12:44:22,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(821): NOT STARTING DFS 2024-11-08T12:44:22,193 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/zookeeper_0, clientPort=55950, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T12:44:22,210 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55950 2024-11-08T12:44:22,226 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:22,230 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:22,405 WARN [Time-limited test {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:22,406 WARN [Time-limited test {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:22,521 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:39526 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775792_1001] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39526 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:22,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775792_1002 (size=7) 2024-11-08T12:44:22,998 WARN [Time-limited test {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:23,009 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e with version=8 2024-11-08T12:44:23,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/hbase-staging 2024-11-08T12:44:23,149 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T12:44:23,440 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7d22a5ea652e:0 server-side Connection retries=45 2024-11-08T12:44:23,452 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:23,452 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:23,459 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T12:44:23,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:23,460 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T12:44:23,660 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T12:44:23,750 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T12:44:23,761 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T12:44:23,765 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T12:44:23,798 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 9633 (auto-detected) 2024-11-08T12:44:23,800 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-08T12:44:23,827 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44263 2024-11-08T12:44:23,858 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44263 connecting to ZooKeeper ensemble=127.0.0.1:55950 2024-11-08T12:44:23,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:442630x0, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T12:44:23,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44263-0x1001f6bebaa0000 connected 2024-11-08T12:44:23,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:23,937 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:23,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:23,952 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e, hbase.cluster.distributed=false 2024-11-08T12:44:23,983 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T12:44:23,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44263 2024-11-08T12:44:23,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44263 2024-11-08T12:44:23,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44263 2024-11-08T12:44:23,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44263 2024-11-08T12:44:23,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44263 2024-11-08T12:44:24,124 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d22a5ea652e:0 server-side Connection retries=45 2024-11-08T12:44:24,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,126 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T12:44:24,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T12:44:24,130 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T12:44:24,132 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T12:44:24,133 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40679 2024-11-08T12:44:24,135 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40679 connecting to ZooKeeper ensemble=127.0.0.1:55950 2024-11-08T12:44:24,136 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:24,139 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:24,152 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406790x0, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:24,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406790x0, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T12:44:24,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40679-0x1001f6bebaa0001 connected 2024-11-08T12:44:24,157 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T12:44:24,170 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T12:44:24,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T12:44:24,179 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T12:44:24,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40679 2024-11-08T12:44:24,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40679 2024-11-08T12:44:24,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40679 2024-11-08T12:44:24,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40679 2024-11-08T12:44:24,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40679 2024-11-08T12:44:24,200 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d22a5ea652e:0 server-side Connection retries=45 2024-11-08T12:44:24,200 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,201 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T12:44:24,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T12:44:24,202 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T12:44:24,202 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T12:44:24,203 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36729 2024-11-08T12:44:24,205 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36729 connecting to ZooKeeper ensemble=127.0.0.1:55950 2024-11-08T12:44:24,206 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:24,209 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:24,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367290x0, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T12:44:24,215 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36729-0x1001f6bebaa0002 connected 2024-11-08T12:44:24,215 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:24,216 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T12:44:24,220 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T12:44:24,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T12:44:24,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T12:44:24,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36729 2024-11-08T12:44:24,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36729 2024-11-08T12:44:24,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36729 2024-11-08T12:44:24,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36729 2024-11-08T12:44:24,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36729 2024-11-08T12:44:24,243 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d22a5ea652e:0 server-side Connection retries=45 2024-11-08T12:44:24,243 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,243 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,243 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T12:44:24,243 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:24,244 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T12:44:24,244 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T12:44:24,244 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T12:44:24,245 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37729 2024-11-08T12:44:24,247 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37729 connecting to ZooKeeper ensemble=127.0.0.1:55950 2024-11-08T12:44:24,249 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:24,251 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:24,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377290x0, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T12:44:24,257 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:377290x0, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:24,257 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37729-0x1001f6bebaa0003 connected 2024-11-08T12:44:24,258 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T12:44:24,259 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T12:44:24,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T12:44:24,262 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T12:44:24,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37729 2024-11-08T12:44:24,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37729 2024-11-08T12:44:24,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37729 2024-11-08T12:44:24,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37729 2024-11-08T12:44:24,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37729 2024-11-08T12:44:24,304 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7d22a5ea652e:44263 2024-11-08T12:44:24,305 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:24,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:24,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:24,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:24,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:24,317 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:24,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:24,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T12:44:24,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:24,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T12:44:24,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:24,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T12:44:24,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:24,353 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T12:44:24,354 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7d22a5ea652e,44263,1731069863207 from backup master directory 2024-11-08T12:44:24,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:24,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:24,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:24,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:24,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:24,359 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T12:44:24,359 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:24,362 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T12:44:24,366 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T12:44:24,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_-9223372036854775788_1002 (size=7) 2024-11-08T12:44:24,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_-9223372036854775789_1002 (size=7) 2024-11-08T12:44:24,423 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/hbase.id] with ID: 3c20363c-411c-40f6-902e-b50f58c985eb 2024-11-08T12:44:24,423 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/.tmp/hbase.id 2024-11-08T12:44:24,434 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:24,434 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:24,441 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:59914 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775776_1003] {}] datanode.DataXceiver(331): 127.0.0.1:39559:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59914 dst: /127.0.0.1:39559 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_-9223372036854775776_1004 (size=42) 2024-11-08T12:44:24,452 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:24,453 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/.tmp/hbase.id]:[hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/hbase.id] 2024-11-08T12:44:24,506 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:24,512 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T12:44:24,534 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-08T12:44:24,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:24,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:24,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:24,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:24,567 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:24,568 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:24,578 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:36416 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775760_1005] {}] datanode.DataXceiver(331): 127.0.0.1:40469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36416 dst: /127.0.0.1:40469 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_-9223372036854775760_1006 (size=196) 2024-11-08T12:44:24,592 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:24,611 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T12:44:24,613 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T12:44:24,619 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T12:44:24,661 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:24,661 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:24,661 WARN [IPC Server handler 4 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:24,662 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:24,666 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,672 WARN [IPC Server handler 1 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:24,672 WARN [IPC Server handler 1 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:24,673 WARN [IPC Server handler 1 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:24,673 WARN [IPC Server handler 1 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:24,674 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000002.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000002.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,682 WARN [IPC Server handler 2 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:24,683 WARN [IPC Server handler 2 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:24,683 WARN [IPC Server handler 2 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:24,683 WARN [IPC Server handler 2 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:24,684 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000003.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000003.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,691 WARN [IPC Server handler 3 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:24,691 WARN [IPC Server handler 3 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:24,691 WARN [IPC Server handler 3 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:24,691 WARN [IPC Server handler 3 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:24,697 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000004.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000004.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,701 WARN [IPC Server handler 0 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:24,701 WARN [IPC Server handler 0 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:24,701 WARN [IPC Server handler 0 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:24,702 WARN [IPC Server handler 0 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:24,702 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000005.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000005.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,706 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:24,706 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:24,707 WARN [IPC Server handler 4 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:24,707 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:24,708 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000006.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000006.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,711 WARN [IPC Server handler 1 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:24,711 WARN [IPC Server handler 1 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:24,712 WARN [IPC Server handler 1 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:24,712 WARN [IPC Server handler 1 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:24,713 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000007.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000007.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,721 WARN [IPC Server handler 2 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:24,721 WARN [IPC Server handler 2 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:24,721 WARN [IPC Server handler 2 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:24,721 WARN [IPC Server handler 2 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:24,722 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000008.1189; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000008.1189 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.region.MasterRegion.bootstrap(MasterRegion.java:241) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:410) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,731 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:24,732 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:24,737 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:39580 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775744_1007] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39580 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775744_1008 (size=1189) 2024-11-08T12:44:24,746 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:24,750 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000001.1189 2024-11-08T12:44:24,751 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000002.1189 2024-11-08T12:44:24,753 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000003.1189 2024-11-08T12:44:24,754 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000004.1189 2024-11-08T12:44:24,755 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000005.1189 2024-11-08T12:44:24,756 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000006.1189 2024-11-08T12:44:24,758 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000007.1189 2024-11-08T12:44:24,759 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.tabledesc/.tableinfo.0000000008.1189 2024-11-08T12:44:24,783 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store 2024-11-08T12:44:24,811 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:24,811 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:24,818 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:39586 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775728_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39586 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:24,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775728_1010 (size=34) 2024-11-08T12:44:24,849 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:24,859 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T12:44:24,864 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:24,865 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T12:44:24,865 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:24,866 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:24,868 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T12:44:24,868 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:24,868 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:24,870 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731069864865Disabling compacts and flushes for region at 1731069864865Disabling writes for close at 1731069864868 (+3 ms)Writing region close event to WAL at 1731069864868Closed at 1731069864868 2024-11-08T12:44:24,872 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/.initializing 2024-11-08T12:44:24,872 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/WALs/7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:24,890 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T12:44:24,923 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C44263%2C1731069863207, suffix=, logDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/WALs/7d22a5ea652e,44263,1731069863207, archiveDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/oldWALs, maxLogs=10 2024-11-08T12:44:24,996 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/WALs/7d22a5ea652e,44263,1731069863207/7d22a5ea652e%2C44263%2C1731069863207.1731069864931, exclude list is [], retry=0 2024-11-08T12:44:25,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:414) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:473) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:468) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:25,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39559,DS-945a1a34-020c-44eb-9686-eccd68d5556b,DISK] 2024-11-08T12:44:25,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40469,DS-b813e00c-15be-4555-9ef3-4c07ec8f3d51,DISK] 2024-11-08T12:44:25,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34381,DS-090cfee0-ba02-4ad2-87d7-fd886a424c14,DISK] 2024-11-08T12:44:25,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T12:44:25,084 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/WALs/7d22a5ea652e,44263,1731069863207/7d22a5ea652e%2C44263%2C1731069863207.1731069864931 2024-11-08T12:44:25,089 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36231:36231),(127.0.0.1/127.0.0.1:41221:41221),(127.0.0.1/127.0.0.1:36629:36629)] 2024-11-08T12:44:25,091 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T12:44:25,095 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:25,103 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,104 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T12:44:25,201 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:25,205 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,209 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T12:44:25,209 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T12:44:25,210 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T12:44:25,214 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T12:44:25,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,220 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T12:44:25,220 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T12:44:25,221 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,226 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,227 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,234 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,235 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,238 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T12:44:25,242 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:25,254 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T12:44:25,256 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74634000, jitterRate=0.11213326454162598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T12:44:25,265 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731069865120Initializing all the Stores at 1731069865123 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069865123Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069865124 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069865125 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069865125Cleaning up temporary data from old regions at 1731069865235 (+110 ms)Region opened successfully at 1731069865265 (+30 ms) 2024-11-08T12:44:25,266 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T12:44:25,306 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df68fe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d22a5ea652e/172.17.0.3:0 2024-11-08T12:44:25,342 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T12:44:25,355 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T12:44:25,355 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T12:44:25,358 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T12:44:25,359 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T12:44:25,365 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T12:44:25,365 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T12:44:25,394 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T12:44:25,405 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T12:44:25,408 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T12:44:25,411 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T12:44:25,415 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T12:44:25,418 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T12:44:25,421 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T12:44:25,425 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T12:44:25,429 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T12:44:25,430 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T12:44:25,432 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T12:44:25,453 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T12:44:25,455 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T12:44:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:25,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,467 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7d22a5ea652e,44263,1731069863207, sessionid=0x1001f6bebaa0000, setting cluster-up flag (Was=false) 2024-11-08T12:44:25,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,510 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T12:44:25,512 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:25,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:25,538 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T12:44:25,541 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:25,554 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T12:44:25,581 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(746): ClusterId : 3c20363c-411c-40f6-902e-b50f58c985eb 2024-11-08T12:44:25,584 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T12:44:25,585 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(746): ClusterId : 3c20363c-411c-40f6-902e-b50f58c985eb 2024-11-08T12:44:25,585 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T12:44:25,591 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T12:44:25,591 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T12:44:25,591 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T12:44:25,591 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T12:44:25,592 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(746): ClusterId : 3c20363c-411c-40f6-902e-b50f58c985eb 2024-11-08T12:44:25,593 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T12:44:25,597 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T12:44:25,597 DEBUG [RS:2;7d22a5ea652e:37729 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f898a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d22a5ea652e/172.17.0.3:0 2024-11-08T12:44:25,601 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T12:44:25,601 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T12:44:25,603 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T12:44:25,605 DEBUG [RS:0;7d22a5ea652e:40679 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b20d8ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d22a5ea652e/172.17.0.3:0 2024-11-08T12:44:25,607 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T12:44:25,608 DEBUG [RS:1;7d22a5ea652e:36729 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663fc719, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d22a5ea652e/172.17.0.3:0 2024-11-08T12:44:25,617 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7d22a5ea652e:37729 2024-11-08T12:44:25,622 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T12:44:25,622 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T12:44:25,622 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T12:44:25,631 DEBUG [RS:0;7d22a5ea652e:40679 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7d22a5ea652e:40679 2024-11-08T12:44:25,631 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T12:44:25,631 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T12:44:25,632 DEBUG [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T12:44:25,635 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,44263,1731069863207 with port=37729, startcode=1731069864242 2024-11-08T12:44:25,635 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,44263,1731069863207 with port=40679, startcode=1731069864083 2024-11-08T12:44:25,640 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7d22a5ea652e:36729 2024-11-08T12:44:25,640 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T12:44:25,640 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T12:44:25,640 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T12:44:25,643 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,44263,1731069863207 with port=36729, startcode=1731069864200 2024-11-08T12:44:25,651 DEBUG [RS:0;7d22a5ea652e:40679 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T12:44:25,651 DEBUG [RS:2;7d22a5ea652e:37729 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T12:44:25,652 DEBUG [RS:1;7d22a5ea652e:36729 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T12:44:25,665 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T12:44:25,678 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T12:44:25,687 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T12:44:25,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T12:44:25,695 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52427, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T12:44:25,695 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37301, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T12:44:25,696 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7d22a5ea652e,44263,1731069863207 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T12:44:25,703 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T12:44:25,705 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7d22a5ea652e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T12:44:25,705 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7d22a5ea652e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T12:44:25,705 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T12:44:25,705 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T12:44:25,705 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7d22a5ea652e:0, corePoolSize=10, maxPoolSize=10 2024-11-08T12:44:25,705 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,706 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T12:44:25,706 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T12:44:25,710 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T12:44:25,731 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T12:44:25,732 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T12:44:25,737 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731069895737 2024-11-08T12:44:25,738 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T12:44:25,738 WARN [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T12:44:25,738 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T12:44:25,738 WARN [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T12:44:25,738 DEBUG [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T12:44:25,738 WARN [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T12:44:25,739 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T12:44:25,740 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T12:44:25,742 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,742 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T12:44:25,745 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T12:44:25,745 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T12:44:25,746 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T12:44:25,746 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T12:44:25,748 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,752 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T12:44:25,753 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T12:44:25,754 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T12:44:25,765 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T12:44:25,765 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T12:44:25,768 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:25,768 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:25,776 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.large.0-1731069865767,5,FailOnTimeoutGroup] 2024-11-08T12:44:25,788 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.small.0-1731069865777,5,FailOnTimeoutGroup] 2024-11-08T12:44:25,789 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,789 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T12:44:25,790 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,790 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,795 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:39610 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775712_1012] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39610 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:25,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775712_1013 (size=1321) 2024-11-08T12:44:25,814 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:25,821 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T12:44:25,821 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e 2024-11-08T12:44:25,831 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:25,831 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:25,840 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,44263,1731069863207 with port=36729, startcode=1731069864200 2024-11-08T12:44:25,842 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:25,845 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,44263,1731069863207 with port=40679, startcode=1731069864083 2024-11-08T12:44:25,845 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] master.ServerManager(517): Registering regionserver=7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:25,849 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,44263,1731069863207 with port=37729, startcode=1731069864242 2024-11-08T12:44:25,850 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:39638 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775696_1014] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39638 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:25,858 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d22a5ea652e,40679,1731069864083 2024-11-08T12:44:25,858 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] master.ServerManager(517): Registering regionserver=7d22a5ea652e,40679,1731069864083 2024-11-08T12:44:25,859 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e 2024-11-08T12:44:25,859 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36053 2024-11-08T12:44:25,859 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T12:44:25,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T12:44:25,865 DEBUG [RS:1;7d22a5ea652e:36729 {}] zookeeper.ZKUtil(111): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:25,866 WARN [RS:1;7d22a5ea652e:36729 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T12:44:25,866 INFO [RS:1;7d22a5ea652e:36729 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T12:44:25,866 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:25,870 DEBUG [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e 2024-11-08T12:44:25,870 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:25,870 DEBUG [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36053 2024-11-08T12:44:25,870 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44263 {}] master.ServerManager(517): Registering regionserver=7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:25,870 DEBUG [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T12:44:25,875 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e 2024-11-08T12:44:25,875 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36053 2024-11-08T12:44:25,875 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T12:44:25,876 DEBUG [RS:0;7d22a5ea652e:40679 {}] zookeeper.ZKUtil(111): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d22a5ea652e,40679,1731069864083 2024-11-08T12:44:25,876 WARN [RS:0;7d22a5ea652e:40679 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T12:44:25,876 INFO [RS:0;7d22a5ea652e:40679 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T12:44:25,876 DEBUG [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,40679,1731069864083 2024-11-08T12:44:25,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775696_1015 (size=32) 2024-11-08T12:44:25,878 WARN [PEWorker-1 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:25,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:25,880 DEBUG [RS:2;7d22a5ea652e:37729 {}] zookeeper.ZKUtil(111): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:25,880 WARN [RS:2;7d22a5ea652e:37729 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T12:44:25,880 INFO [RS:2;7d22a5ea652e:37729 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T12:44:25,881 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:25,883 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d22a5ea652e,37729,1731069864242] 2024-11-08T12:44:25,884 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d22a5ea652e,36729,1731069864200] 2024-11-08T12:44:25,884 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d22a5ea652e,40679,1731069864083] 2024-11-08T12:44:25,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T12:44:25,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T12:44:25,913 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:25,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T12:44:25,926 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T12:44:25,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:25,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T12:44:25,930 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T12:44:25,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,931 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:25,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T12:44:25,933 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T12:44:25,933 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T12:44:25,933 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T12:44:25,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T12:44:25,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:25,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:25,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T12:44:25,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740 2024-11-08T12:44:25,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740 2024-11-08T12:44:25,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T12:44:25,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T12:44:25,947 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T12:44:25,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T12:44:25,957 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T12:44:25,957 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T12:44:25,965 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=2) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:25,965 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:25,965 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=1) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:25,965 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:25,966 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T12:44:25,966 INFO [RS:0;7d22a5ea652e:40679 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T12:44:25,966 INFO [RS:2;7d22a5ea652e:37729 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T12:44:25,966 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,966 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,965 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:25,967 INFO [RS:1;7d22a5ea652e:36729 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T12:44:25,967 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,967 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:25,968 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T12:44:25,968 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T12:44:25,968 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=2) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:25,968 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:25,968 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=1) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:25,968 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:25,968 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:25,969 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:25,975 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T12:44:25,975 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T12:44:25,976 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,976 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,976 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T12:44:25,977 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,977 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d22a5ea652e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T12:44:25,978 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d22a5ea652e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T12:44:25,978 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,978 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,981 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,982 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:25,982 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:25,982 DEBUG [RS:0;7d22a5ea652e:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:25,982 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T12:44:25,982 DEBUG [RS:2;7d22a5ea652e:37729 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:25,982 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,982 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,982 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,982 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,982 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,982 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,983 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d22a5ea652e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T12:44:25,983 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,983 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,983 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,983 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,983 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,983 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:25,983 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:25,984 DEBUG [RS:1;7d22a5ea652e:36729 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:25,986 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T12:44:25,987 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70289165, jitterRate=0.047390177845954895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T12:44:25,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731069865880Initializing all the Stores at 1731069865883 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069865884 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069865908 (+24 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069865909 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069865909Cleaning up temporary data from old regions at 1731069865946 (+37 ms)Region opened successfully at 1731069865989 (+43 ms) 2024-11-08T12:44:25,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T12:44:25,989 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T12:44:25,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T12:44:25,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T12:44:25,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T12:44:25,992 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,993 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,993 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,993 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,993 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:25,993 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,40679,1731069864083-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T12:44:26,002 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,002 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,002 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,002 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,002 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,002 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,36729,1731069864200-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T12:44:26,002 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T12:44:26,002 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,003 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731069865989Disabling compacts and flushes for region at 1731069865989Disabling writes for close at 1731069865990 (+1 ms)Writing region close event to WAL at 1731069866002 (+12 ms)Closed at 1731069866002 2024-11-08T12:44:26,003 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,003 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,003 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,003 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,003 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,37729,1731069864242-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T12:44:26,013 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T12:44:26,013 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T12:44:26,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T12:44:26,031 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T12:44:26,031 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T12:44:26,034 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,36729,1731069864200-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,035 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,035 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.Replication(171): 7d22a5ea652e,36729,1731069864200 started 2024-11-08T12:44:26,035 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,40679,1731069864083-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,035 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,036 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.Replication(171): 7d22a5ea652e,40679,1731069864083 started 2024-11-08T12:44:26,041 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T12:44:26,043 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T12:44:26,043 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,37729,1731069864242-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,044 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,044 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.Replication(171): 7d22a5ea652e,37729,1731069864242 started 2024-11-08T12:44:26,050 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T12:44:26,070 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,070 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(1482): Serving as 7d22a5ea652e,40679,1731069864083, RpcServer on 7d22a5ea652e/172.17.0.3:40679, sessionid=0x1001f6bebaa0001 2024-11-08T12:44:26,071 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T12:44:26,071 DEBUG [RS:0;7d22a5ea652e:40679 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d22a5ea652e,40679,1731069864083 2024-11-08T12:44:26,072 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,40679,1731069864083' 2024-11-08T12:44:26,072 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T12:44:26,073 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T12:44:26,074 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,074 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1482): Serving as 7d22a5ea652e,37729,1731069864242, RpcServer on 7d22a5ea652e/172.17.0.3:37729, sessionid=0x1001f6bebaa0003 2024-11-08T12:44:26,074 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T12:44:26,074 DEBUG [RS:2;7d22a5ea652e:37729 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:26,074 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T12:44:26,074 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,37729,1731069864242' 2024-11-08T12:44:26,074 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T12:44:26,074 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T12:44:26,075 DEBUG [RS:0;7d22a5ea652e:40679 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d22a5ea652e,40679,1731069864083 2024-11-08T12:44:26,075 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,40679,1731069864083' 2024-11-08T12:44:26,075 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T12:44:26,075 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T12:44:26,076 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T12:44:26,077 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T12:44:26,077 DEBUG [RS:2;7d22a5ea652e:37729 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:26,077 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,37729,1731069864242' 2024-11-08T12:44:26,077 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T12:44:26,077 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T12:44:26,078 DEBUG [RS:2;7d22a5ea652e:37729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T12:44:26,078 INFO [RS:2;7d22a5ea652e:37729 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T12:44:26,079 INFO [RS:2;7d22a5ea652e:37729 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T12:44:26,079 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T12:44:26,079 DEBUG [RS:0;7d22a5ea652e:40679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T12:44:26,080 INFO [RS:0;7d22a5ea652e:40679 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T12:44:26,080 INFO [RS:0;7d22a5ea652e:40679 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T12:44:26,085 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:26,085 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1482): Serving as 7d22a5ea652e,36729,1731069864200, RpcServer on 7d22a5ea652e/172.17.0.3:36729, sessionid=0x1001f6bebaa0002 2024-11-08T12:44:26,085 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T12:44:26,085 DEBUG [RS:1;7d22a5ea652e:36729 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:26,085 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,36729,1731069864200' 2024-11-08T12:44:26,085 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T12:44:26,086 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T12:44:26,087 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T12:44:26,087 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T12:44:26,087 DEBUG [RS:1;7d22a5ea652e:36729 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:26,087 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,36729,1731069864200' 2024-11-08T12:44:26,087 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T12:44:26,088 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T12:44:26,089 DEBUG [RS:1;7d22a5ea652e:36729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T12:44:26,089 INFO [RS:1;7d22a5ea652e:36729 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T12:44:26,089 INFO [RS:1;7d22a5ea652e:36729 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T12:44:26,184 INFO [RS:0;7d22a5ea652e:40679 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T12:44:26,184 INFO [RS:2;7d22a5ea652e:37729 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T12:44:26,187 INFO [RS:0;7d22a5ea652e:40679 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C40679%2C1731069864083, suffix=, logDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,40679,1731069864083, archiveDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/oldWALs, maxLogs=32 2024-11-08T12:44:26,187 INFO [RS:2;7d22a5ea652e:37729 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C37729%2C1731069864242, suffix=, logDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,37729,1731069864242, archiveDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/oldWALs, maxLogs=32 2024-11-08T12:44:26,189 INFO [RS:1;7d22a5ea652e:36729 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T12:44:26,192 INFO [RS:1;7d22a5ea652e:36729 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C36729%2C1731069864200, suffix=, logDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,36729,1731069864200, archiveDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/oldWALs, maxLogs=32 2024-11-08T12:44:26,201 WARN [7d22a5ea652e:44263 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T12:44:26,207 DEBUG [RS:0;7d22a5ea652e:40679 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,40679,1731069864083/7d22a5ea652e%2C40679%2C1731069864083.1731069866192, exclude list is [], retry=0 2024-11-08T12:44:26,209 WARN [IPC Server handler 3 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:26,209 WARN [IPC Server handler 3 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:26,209 WARN [IPC Server handler 3 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:26,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40469,DS-b813e00c-15be-4555-9ef3-4c07ec8f3d51,DISK] 2024-11-08T12:44:26,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39559,DS-945a1a34-020c-44eb-9686-eccd68d5556b,DISK] 2024-11-08T12:44:26,213 DEBUG [RS:2;7d22a5ea652e:37729 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,37729,1731069864242/7d22a5ea652e%2C37729%2C1731069864242.1731069866192, exclude list is [], retry=0 2024-11-08T12:44:26,214 DEBUG [RS:1;7d22a5ea652e:36729 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,36729,1731069864200/7d22a5ea652e%2C36729%2C1731069864200.1731069866194, exclude list is [], retry=0 2024-11-08T12:44:26,216 INFO [RS:0;7d22a5ea652e:40679 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,40679,1731069864083/7d22a5ea652e%2C40679%2C1731069864083.1731069866192 2024-11-08T12:44:26,216 DEBUG [RS:0;7d22a5ea652e:40679 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36629:36629),(127.0.0.1/127.0.0.1:41221:41221)] 2024-11-08T12:44:26,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34381,DS-090cfee0-ba02-4ad2-87d7-fd886a424c14,DISK] 2024-11-08T12:44:26,238 WARN [IPC Server handler 1 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:26,238 WARN [IPC Server handler 1 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:26,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39559,DS-945a1a34-020c-44eb-9686-eccd68d5556b,DISK] 2024-11-08T12:44:26,238 WARN [IPC Server handler 1 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:26,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40469,DS-b813e00c-15be-4555-9ef3-4c07ec8f3d51,DISK] 2024-11-08T12:44:26,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39559,DS-945a1a34-020c-44eb-9686-eccd68d5556b,DISK] 2024-11-08T12:44:26,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40469,DS-b813e00c-15be-4555-9ef3-4c07ec8f3d51,DISK] 2024-11-08T12:44:26,246 INFO [RS:2;7d22a5ea652e:37729 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,37729,1731069864242/7d22a5ea652e%2C37729%2C1731069864242.1731069866192 2024-11-08T12:44:26,248 INFO [RS:1;7d22a5ea652e:36729 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,36729,1731069864200/7d22a5ea652e%2C36729%2C1731069864200.1731069866194 2024-11-08T12:44:26,249 DEBUG [RS:2;7d22a5ea652e:37729 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36231:36231),(127.0.0.1/127.0.0.1:36629:36629),(127.0.0.1/127.0.0.1:41221:41221)] 2024-11-08T12:44:26,249 DEBUG [RS:1;7d22a5ea652e:36729 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36629:36629),(127.0.0.1/127.0.0.1:41221:41221)] 2024-11-08T12:44:26,454 DEBUG [7d22a5ea652e:44263 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-08T12:44:26,463 DEBUG [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(204): Hosts are {7d22a5ea652e=0} racks are {/default-rack=0} 2024-11-08T12:44:26,470 DEBUG [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T12:44:26,471 DEBUG [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T12:44:26,471 DEBUG [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T12:44:26,471 DEBUG [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T12:44:26,471 DEBUG [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T12:44:26,471 DEBUG [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T12:44:26,471 INFO [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T12:44:26,471 INFO [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T12:44:26,471 INFO [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T12:44:26,471 DEBUG [7d22a5ea652e:44263 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T12:44:26,479 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:26,486 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d22a5ea652e,37729,1731069864242, state=OPENING 2024-11-08T12:44:26,491 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T12:44:26,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:26,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:26,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:26,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:26,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:26,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:26,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:26,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:26,496 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T12:44:26,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7d22a5ea652e,37729,1731069864242}] 2024-11-08T12:44:26,680 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T12:44:26,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T12:44:26,696 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T12:44:26,697 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T12:44:26,698 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T12:44:26,703 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C37729%2C1731069864242.meta, suffix=.meta, logDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,37729,1731069864242, archiveDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/oldWALs, maxLogs=32 2024-11-08T12:44:26,724 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,37729,1731069864242/7d22a5ea652e%2C37729%2C1731069864242.meta.1731069866705.meta, exclude list is [], retry=0 2024-11-08T12:44:26,727 WARN [IPC Server handler 0 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:26,727 WARN [IPC Server handler 0 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:26,727 WARN [IPC Server handler 0 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:26,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40469,DS-b813e00c-15be-4555-9ef3-4c07ec8f3d51,DISK] 2024-11-08T12:44:26,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39559,DS-945a1a34-020c-44eb-9686-eccd68d5556b,DISK] 2024-11-08T12:44:26,733 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/WALs/7d22a5ea652e,37729,1731069864242/7d22a5ea652e%2C37729%2C1731069864242.meta.1731069866705.meta 2024-11-08T12:44:26,733 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41221:41221),(127.0.0.1/127.0.0.1:36629:36629)] 2024-11-08T12:44:26,733 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T12:44:26,735 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T12:44:26,739 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T12:44:26,745 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T12:44:26,751 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T12:44:26,752 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:26,752 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T12:44:26,753 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T12:44:26,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T12:44:26,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T12:44:26,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:26,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:26,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T12:44:26,761 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T12:44:26,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:26,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:26,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T12:44:26,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T12:44:26,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:26,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:26,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T12:44:26,766 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T12:44:26,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:26,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:26,767 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T12:44:26,769 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740 2024-11-08T12:44:26,772 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740 2024-11-08T12:44:26,774 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T12:44:26,774 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T12:44:26,775 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T12:44:26,778 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T12:44:26,779 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67868901, jitterRate=0.011325433850288391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T12:44:26,780 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T12:44:26,781 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731069866753Writing region info on filesystem at 1731069866753Initializing all the Stores at 1731069866756 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069866756Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069866756Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069866756Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069866756Cleaning up temporary data from old regions at 1731069866775 (+19 ms)Running coprocessor post-open hooks at 1731069866780 (+5 ms)Region opened successfully at 1731069866781 (+1 ms) 2024-11-08T12:44:26,789 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731069866667 2024-11-08T12:44:26,828 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T12:44:26,828 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T12:44:26,830 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:26,832 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d22a5ea652e,37729,1731069864242, state=OPEN 2024-11-08T12:44:26,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T12:44:26,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T12:44:26,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T12:44:26,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T12:44:26,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:26,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:26,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:26,835 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:26,836 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:26,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T12:44:26,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d22a5ea652e,37729,1731069864242 in 339 msec 2024-11-08T12:44:26,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T12:44:26,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 821 msec 2024-11-08T12:44:26,850 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T12:44:26,850 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T12:44:26,881 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T12:44:26,882 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d22a5ea652e,37729,1731069864242, seqNum=-1] 2024-11-08T12:44:26,902 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T12:44:26,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35349, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T12:44:26,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3350 sec 2024-11-08T12:44:26,943 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731069866943, completionTime=-1 2024-11-08T12:44:26,947 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-08T12:44:26,947 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T12:44:26,995 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-08T12:44:26,996 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731069926996 2024-11-08T12:44:26,996 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731069986996 2024-11-08T12:44:26,996 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 48 msec 2024-11-08T12:44:26,998 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T12:44:27,010 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,44263,1731069863207-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:27,011 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,44263,1731069863207-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:27,011 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,44263,1731069863207-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:27,013 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7d22a5ea652e:44263, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:27,014 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:27,015 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:27,033 DEBUG [master/7d22a5ea652e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T12:44:27,062 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.701sec 2024-11-08T12:44:27,064 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T12:44:27,065 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T12:44:27,067 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T12:44:27,067 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T12:44:27,067 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T12:44:27,068 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,44263,1731069863207-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T12:44:27,069 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,44263,1731069863207-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T12:44:27,074 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T12:44:27,075 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T12:44:27,076 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,44263,1731069863207-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:27,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127f4194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T12:44:27,101 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T12:44:27,101 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T12:44:27,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d22a5ea652e,44263,-1 for getting cluster id 2024-11-08T12:44:27,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T12:44:27,116 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3c20363c-411c-40f6-902e-b50f58c985eb' 2024-11-08T12:44:27,119 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T12:44:27,119 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3c20363c-411c-40f6-902e-b50f58c985eb" 2024-11-08T12:44:27,119 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2739c676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T12:44:27,119 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d22a5ea652e,44263,-1] 2024-11-08T12:44:27,122 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T12:44:27,124 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:27,125 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T12:44:27,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42ed93ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T12:44:27,129 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T12:44:27,137 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d22a5ea652e,37729,1731069864242, seqNum=-1] 2024-11-08T12:44:27,138 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T12:44:27,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T12:44:27,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:27,166 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T12:44:27,171 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:27,175 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@340addf1 2024-11-08T12:44:27,176 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T12:44:27,179 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T12:44:27,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T12:44:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC 2024-11-08T12:44:27,198 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T12:44:27,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestHBaseWalOnEC" procId is: 4 2024-11-08T12:44:27,201 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:27,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T12:44:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T12:44:27,212 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyRackFaultTolerant(145): Only able to place 2 of total expected 3 (maxNodesPerRack=3, numOfReplicas=3) nodes evenly across racks, falling back to evenly place on the remaining racks. This may not guarantee rack-level fault tolerance. Please check if the racks are configured properly. 2024-11-08T12:44:27,212 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:27,212 WARN [IPC Server handler 4 on default port 36053 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:27,212 WARN [IPC Server handler 4 on default port 36053 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:27,214 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(635): Failed write hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/.tabledesc/.tableinfo.0000000001.392; retrying up to 10 times org.apache.hadoop.ipc.RemoteException: File /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/.tabledesc/.tableinfo.0000000001.392 could only be written to 2 of the 3 required nodes for RS-3-2-1024k. There are 3 datanode(s) running and 3 node(s) are excluded in this operation. at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2480) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:293) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:3075) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:932) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:603) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$addBlock$11(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:500) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy47.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor6.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy48.addBlock(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1143) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.allocateNewBlock(DFSStripedOutputStream.java:508) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.writeChunk(DFSStripedOutputStream.java:561) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:220) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:165) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:146) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSStripedOutputStream.closeImpl(DFSStripedOutputStream.java:1234) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.util.FSTableDescriptors.writeTableDescriptor(FSTableDescriptors.java:631) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:707) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FSTableDescriptors.createTableDescriptorForTableDirectory(FSTableDescriptors.java:680) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.createFsLayout(CreateTableProcedure.java:376) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.createFsLayout(CreateTableProcedure.java:353) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.executeFromState(CreateTableProcedure.java:107) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.executeFromState(CreateTableProcedure.java:59) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T12:44:27,223 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:27,223 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:27,227 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:45640 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775680_1020] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45640 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:27,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775680_1021 (size=392) 2024-11-08T12:44:27,235 WARN [PEWorker-3 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:27,237 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/.tabledesc/.tableinfo.0000000001.392 2024-11-08T12:44:27,238 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e0d37739fbb69b40157810d458be9c4a, NAME => 'TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e 2024-11-08T12:44:27,246 WARN [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:27,246 WARN [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:27,252 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:45658 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775664_1022] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45658 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:27,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775664_1023 (size=51) 2024-11-08T12:44:27,263 WARN [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:27,263 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:27,263 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1722): Closing e0d37739fbb69b40157810d458be9c4a, disabling compactions & flushes 2024-11-08T12:44:27,263 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:27,263 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:27,263 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. after waiting 0 ms 2024-11-08T12:44:27,263 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:27,263 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:27,264 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1676): Region close journal for e0d37739fbb69b40157810d458be9c4a: Waiting for close lock at 1731069867263Disabling compacts and flushes for region at 1731069867263Disabling writes for close at 1731069867263Writing region close event to WAL at 1731069867263Closed at 1731069867263 2024-11-08T12:44:27,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T12:44:27,273 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1731069867266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731069867266"}]},"ts":"1731069867266"} 2024-11-08T12:44:27,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_-9223372036854775773_1004 (size=42) 2024-11-08T12:44:27,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_-9223372036854775757_1006 (size=196) 2024-11-08T12:44:27,287 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T12:44:27,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_-9223372036854775741_1008 (size=1189) 2024-11-08T12:44:27,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_-9223372036854775740_1008 (size=1189) 2024-11-08T12:44:27,290 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T12:44:27,293 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731069867290"}]},"ts":"1731069867290"} 2024-11-08T12:44:27,298 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLING in hbase:meta 2024-11-08T12:44:27,299 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {7d22a5ea652e=0} racks are {/default-rack=0} 2024-11-08T12:44:27,300 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T12:44:27,300 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T12:44:27,300 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T12:44:27,300 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T12:44:27,300 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T12:44:27,300 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T12:44:27,300 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T12:44:27,300 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T12:44:27,300 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T12:44:27,300 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T12:44:27,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=e0d37739fbb69b40157810d458be9c4a, ASSIGN}] 2024-11-08T12:44:27,304 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=e0d37739fbb69b40157810d458be9c4a, ASSIGN 2024-11-08T12:44:27,307 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=e0d37739fbb69b40157810d458be9c4a, ASSIGN; state=OFFLINE, location=7d22a5ea652e,36729,1731069864200; forceNewPlan=false, retain=false 2024-11-08T12:44:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T12:44:27,460 INFO [7d22a5ea652e:44263 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T12:44:27,461 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e0d37739fbb69b40157810d458be9c4a, regionState=OPENING, regionLocation=7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:27,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=e0d37739fbb69b40157810d458be9c4a, ASSIGN because future has completed 2024-11-08T12:44:27,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0d37739fbb69b40157810d458be9c4a, server=7d22a5ea652e,36729,1731069864200}] 2024-11-08T12:44:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T12:44:27,623 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T12:44:27,625 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33345, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T12:44:27,634 INFO [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:27,635 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e0d37739fbb69b40157810d458be9c4a, NAME => 'TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a.', STARTKEY => '', ENDKEY => ''} 2024-11-08T12:44:27,636 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestHBaseWalOnEC e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,636 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:27,636 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,636 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,641 INFO [StoreOpener-e0d37739fbb69b40157810d458be9c4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,647 INFO [StoreOpener-e0d37739fbb69b40157810d458be9c4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0d37739fbb69b40157810d458be9c4a columnFamilyName cf 2024-11-08T12:44:27,648 DEBUG [StoreOpener-e0d37739fbb69b40157810d458be9c4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:27,649 INFO [StoreOpener-e0d37739fbb69b40157810d458be9c4a-1 {}] regionserver.HStore(327): Store=e0d37739fbb69b40157810d458be9c4a/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T12:44:27,649 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,655 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,656 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,659 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,659 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,664 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,677 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T12:44:27,678 INFO [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e0d37739fbb69b40157810d458be9c4a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61295615, jitterRate=-0.0866241604089737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T12:44:27,678 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:27,680 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e0d37739fbb69b40157810d458be9c4a: Running coprocessor pre-open hook at 1731069867637Writing region info on filesystem at 1731069867637Initializing all the Stores at 1731069867641 (+4 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069867641Cleaning up temporary data from old regions at 1731069867659 (+18 ms)Running coprocessor post-open hooks at 1731069867678 (+19 ms)Region opened successfully at 1731069867679 (+1 ms) 2024-11-08T12:44:27,687 INFO [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a., pid=6, masterSystemTime=1731069867622 2024-11-08T12:44:27,694 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:27,694 INFO [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:27,695 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e0d37739fbb69b40157810d458be9c4a, regionState=OPEN, openSeqNum=2, regionLocation=7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:27,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e0d37739fbb69b40157810d458be9c4a, server=7d22a5ea652e,36729,1731069864200 because future has completed 2024-11-08T12:44:27,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T12:44:27,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e0d37739fbb69b40157810d458be9c4a, server=7d22a5ea652e,36729,1731069864200 in 241 msec 2024-11-08T12:44:27,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T12:44:27,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=e0d37739fbb69b40157810d458be9c4a, ASSIGN in 414 msec 2024-11-08T12:44:27,726 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T12:44:27,726 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731069867726"}]},"ts":"1731069867726"} 2024-11-08T12:44:27,730 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLED in hbase:meta 2024-11-08T12:44:27,732 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T12:44:27,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC in 544 msec 2024-11-08T12:44:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T12:44:27,836 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestHBaseWalOnEC completed 2024-11-08T12:44:27,836 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table TestHBaseWalOnEC get assigned. Timeout = 60000ms 2024-11-08T12:44:27,837 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T12:44:27,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table TestHBaseWalOnEC assigned to meta. Checking AM states. 2024-11-08T12:44:27,845 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T12:44:27,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table TestHBaseWalOnEC assigned. 2024-11-08T12:44:27,856 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestHBaseWalOnEC', row='row', locateType=CURRENT is [region=TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a., hostname=7d22a5ea652e,36729,1731069864200, seqNum=2] 2024-11-08T12:44:27,858 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T12:44:27,860 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T12:44:27,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestHBaseWalOnEC 2024-11-08T12:44:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC 2024-11-08T12:44:27,884 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_PREPARE 2024-11-08T12:44:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T12:44:27,887 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T12:44:27,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T12:44:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T12:44:28,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36729 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T12:44:28,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:28,061 INFO [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e0d37739fbb69b40157810d458be9c4a 1/1 column families, dataSize=32 B heapSize=360 B 2024-11-08T12:44:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a/.tmp/cf/35cc48d3482b4ff1b1c7df7336336b1a is 36, key is row/cf:cq/1731069867861/Put/seqid=0 2024-11-08T12:44:28,151 WARN [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:28,151 WARN [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:28,170 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_46941383_22 at /127.0.0.1:45694 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775648_1024] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45694 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:28,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775648_1025 (size=4787) 2024-11-08T12:44:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T12:44:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T12:44:28,588 WARN [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:28,589 INFO [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a/.tmp/cf/35cc48d3482b4ff1b1c7df7336336b1a 2024-11-08T12:44:28,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a/.tmp/cf/35cc48d3482b4ff1b1c7df7336336b1a as hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a/cf/35cc48d3482b4ff1b1c7df7336336b1a 2024-11-08T12:44:28,660 INFO [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a/cf/35cc48d3482b4ff1b1c7df7336336b1a, entries=1, sequenceid=5, filesize=4.7 K 2024-11-08T12:44:28,669 INFO [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~32 B/32, heapSize ~344 B/344, currentSize=0 B/0 for e0d37739fbb69b40157810d458be9c4a in 607ms, sequenceid=5, compaction requested=false 2024-11-08T12:44:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestHBaseWalOnEC' 2024-11-08T12:44:28,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e0d37739fbb69b40157810d458be9c4a: 2024-11-08T12:44:28,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:28,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-08T12:44:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-08T12:44:28,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T12:44:28,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 794 msec 2024-11-08T12:44:28,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC in 819 msec 2024-11-08T12:44:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44263 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T12:44:29,025 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestHBaseWalOnEC completed 2024-11-08T12:44:29,041 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T12:44:29,041 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T12:44:29,041 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:29,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T12:44:29,048 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T12:44:29,048 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=523817283, stopped=false 2024-11-08T12:44:29,048 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7d22a5ea652e,44263,1731069863207 2024-11-08T12:44:29,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:29,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:29,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:29,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:29,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:29,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:29,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:29,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:29,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T12:44:29,051 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T12:44:29,051 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:29,051 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:29,051 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:29,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,052 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d22a5ea652e,40679,1731069864083' ***** 2024-11-08T12:44:29,052 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T12:44:29,052 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d22a5ea652e,36729,1731069864200' ***** 2024-11-08T12:44:29,052 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T12:44:29,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:29,052 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d22a5ea652e,37729,1731069864242' ***** 2024-11-08T12:44:29,052 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T12:44:29,052 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T12:44:29,052 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T12:44:29,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:29,053 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T12:44:29,053 INFO [RS:0;7d22a5ea652e:40679 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T12:44:29,053 INFO [RS:2;7d22a5ea652e:37729 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T12:44:29,053 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T12:44:29,053 INFO [RS:2;7d22a5ea652e:37729 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T12:44:29,053 INFO [RS:0;7d22a5ea652e:40679 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T12:44:29,053 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(959): stopping server 7d22a5ea652e,40679,1731069864083 2024-11-08T12:44:29,053 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(959): stopping server 7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:29,053 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T12:44:29,054 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T12:44:29,054 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T12:44:29,054 INFO [RS:1;7d22a5ea652e:36729 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T12:44:29,054 INFO [RS:0;7d22a5ea652e:40679 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7d22a5ea652e:40679. 2024-11-08T12:44:29,054 INFO [RS:2;7d22a5ea652e:37729 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7d22a5ea652e:37729. 2024-11-08T12:44:29,054 INFO [RS:1;7d22a5ea652e:36729 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T12:44:29,054 DEBUG [RS:0;7d22a5ea652e:40679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:29,054 DEBUG [RS:2;7d22a5ea652e:37729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:29,054 DEBUG [RS:0;7d22a5ea652e:40679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,054 DEBUG [RS:2;7d22a5ea652e:37729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,054 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(3091): Received CLOSE for e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:29,054 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(976): stopping server 7d22a5ea652e,40679,1731069864083; all regions closed. 2024-11-08T12:44:29,054 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T12:44:29,054 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T12:44:29,054 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T12:44:29,054 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T12:44:29,055 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T12:44:29,055 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(959): stopping server 7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:29,055 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T12:44:29,055 INFO [RS:1;7d22a5ea652e:36729 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7d22a5ea652e:36729. 2024-11-08T12:44:29,055 DEBUG [RS:1;7d22a5ea652e:36729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:29,055 DEBUG [RS:1;7d22a5ea652e:36729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,055 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T12:44:29,056 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T12:44:29,055 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e0d37739fbb69b40157810d458be9c4a, disabling compactions & flushes 2024-11-08T12:44:29,055 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T12:44:29,056 INFO [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:29,056 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T12:44:29,056 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1325): Online Regions={e0d37739fbb69b40157810d458be9c4a=TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a.} 2024-11-08T12:44:29,056 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T12:44:29,056 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T12:44:29,056 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:29,056 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T12:44:29,056 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. after waiting 0 ms 2024-11-08T12:44:29,056 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:29,056 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T12:44:29,056 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T12:44:29,056 DEBUG [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1351): Waiting on e0d37739fbb69b40157810d458be9c4a 2024-11-08T12:44:29,056 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB 2024-11-08T12:44:29,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_1073741826_1016 (size=93) 2024-11-08T12:44:29,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741826_1016 (size=93) 2024-11-08T12:44:29,070 DEBUG [RS:0;7d22a5ea652e:40679 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/oldWALs 2024-11-08T12:44:29,070 INFO [RS:0;7d22a5ea652e:40679 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d22a5ea652e%2C40679%2C1731069864083:(num 1731069866192) 2024-11-08T12:44:29,070 DEBUG [RS:0;7d22a5ea652e:40679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,070 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:29,070 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T12:44:29,070 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.ChoreService(370): Chore service for: regionserver/7d22a5ea652e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T12:44:29,071 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T12:44:29,071 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T12:44:29,071 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T12:44:29,071 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T12:44:29,071 INFO [regionserver/7d22a5ea652e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T12:44:29,071 INFO [RS:0;7d22a5ea652e:40679 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40679 2024-11-08T12:44:29,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d22a5ea652e,40679,1731069864083 2024-11-08T12:44:29,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T12:44:29,075 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T12:44:29,076 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d22a5ea652e,40679,1731069864083] 2024-11-08T12:44:29,078 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d22a5ea652e,40679,1731069864083 already deleted, retry=false 2024-11-08T12:44:29,078 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d22a5ea652e,40679,1731069864083 expired; onlineServers=2 2024-11-08T12:44:29,080 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/default/TestHBaseWalOnEC/e0d37739fbb69b40157810d458be9c4a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T12:44:29,083 INFO [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:29,083 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e0d37739fbb69b40157810d458be9c4a: Waiting for close lock at 1731069869055Running coprocessor pre-close hooks at 1731069869055Disabling compacts and flushes for region at 1731069869055Disabling writes for close at 1731069869056 (+1 ms)Writing region close event to WAL at 1731069869058 (+2 ms)Running coprocessor post-close hooks at 1731069869081 (+23 ms)Closed at 1731069869083 (+2 ms) 2024-11-08T12:44:29,084 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a. 2024-11-08T12:44:29,099 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/info/f513572e57b148f1bfcecdf46e08625e is 153, key is TestHBaseWalOnEC,,1731069867180.e0d37739fbb69b40157810d458be9c4a./info:regioninfo/1731069867695/Put/seqid=0 2024-11-08T12:44:29,103 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,103 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,110 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1335677681_22 at /127.0.0.1:33062 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775632_1026] {}] datanode.DataXceiver(331): 127.0.0.1:40469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33062 dst: /127.0.0.1:40469 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:29,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_-9223372036854775632_1027 (size=6637) 2024-11-08T12:44:29,125 INFO [regionserver/7d22a5ea652e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:29,125 INFO [regionserver/7d22a5ea652e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:29,125 INFO [regionserver/7d22a5ea652e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:29,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:29,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x1001f6bebaa0001, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:29,179 INFO [RS:0;7d22a5ea652e:40679 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T12:44:29,179 INFO [RS:0;7d22a5ea652e:40679 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d22a5ea652e,40679,1731069864083; zookeeper connection closed. 2024-11-08T12:44:29,179 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@268d0ebc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@268d0ebc 2024-11-08T12:44:29,256 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T12:44:29,256 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(976): stopping server 7d22a5ea652e,36729,1731069864200; all regions closed. 2024-11-08T12:44:29,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_1073741828_1018 (size=1298) 2024-11-08T12:44:29,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741828_1018 (size=1298) 2024-11-08T12:44:29,265 DEBUG [RS:1;7d22a5ea652e:36729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/oldWALs 2024-11-08T12:44:29,265 INFO [RS:1;7d22a5ea652e:36729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d22a5ea652e%2C36729%2C1731069864200:(num 1731069866194) 2024-11-08T12:44:29,265 DEBUG [RS:1;7d22a5ea652e:36729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,265 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:29,265 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T12:44:29,265 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.ChoreService(370): Chore service for: regionserver/7d22a5ea652e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T12:44:29,266 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T12:44:29,266 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T12:44:29,266 INFO [regionserver/7d22a5ea652e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T12:44:29,266 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T12:44:29,266 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T12:44:29,266 INFO [RS:1;7d22a5ea652e:36729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36729 2024-11-08T12:44:29,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d22a5ea652e,36729,1731069864200 2024-11-08T12:44:29,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T12:44:29,268 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T12:44:29,269 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d22a5ea652e,36729,1731069864200] 2024-11-08T12:44:29,271 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d22a5ea652e,36729,1731069864200 already deleted, retry=false 2024-11-08T12:44:29,271 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d22a5ea652e,36729,1731069864200 expired; onlineServers=1 2024-11-08T12:44:29,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:29,370 INFO [RS:1;7d22a5ea652e:36729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T12:44:29,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36729-0x1001f6bebaa0002, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:29,370 INFO [RS:1;7d22a5ea652e:36729 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d22a5ea652e,36729,1731069864200; zookeeper connection closed. 2024-11-08T12:44:29,371 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a5f3e7d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a5f3e7d 2024-11-08T12:44:29,457 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T12:44:29,518 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:29,518 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/info/f513572e57b148f1bfcecdf46e08625e 2024-11-08T12:44:29,551 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/ns/afa02406bbbb4861a8f3741fcd49c540 is 43, key is default/ns:d/1731069866910/Put/seqid=0 2024-11-08T12:44:29,554 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,554 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,559 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1335677681_22 at /127.0.0.1:45702 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775616_1028] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45702 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:29,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775616_1029 (size=5153) 2024-11-08T12:44:29,564 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:29,564 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/ns/afa02406bbbb4861a8f3741fcd49c540 2024-11-08T12:44:29,595 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/table/a90c7dafe9a548ca9fb35b3918bbebff is 52, key is TestHBaseWalOnEC/table:state/1731069867726/Put/seqid=0 2024-11-08T12:44:29,598 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,598 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,614 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1335677681_22 at /127.0.0.1:45984 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775600_1030] {}] datanode.DataXceiver(331): 127.0.0.1:39559:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45984 dst: /127.0.0.1:39559 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:29,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_-9223372036854775600_1031 (size=5249) 2024-11-08T12:44:29,627 WARN [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:29,627 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=96 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/table/a90c7dafe9a548ca9fb35b3918bbebff 2024-11-08T12:44:29,646 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/info/f513572e57b148f1bfcecdf46e08625e as hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/info/f513572e57b148f1bfcecdf46e08625e 2024-11-08T12:44:29,657 DEBUG [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T12:44:29,660 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/info/f513572e57b148f1bfcecdf46e08625e, entries=10, sequenceid=11, filesize=6.5 K 2024-11-08T12:44:29,665 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/ns/afa02406bbbb4861a8f3741fcd49c540 as hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/ns/afa02406bbbb4861a8f3741fcd49c540 2024-11-08T12:44:29,706 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/ns/afa02406bbbb4861a8f3741fcd49c540, entries=2, sequenceid=11, filesize=5.0 K 2024-11-08T12:44:29,713 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/.tmp/table/a90c7dafe9a548ca9fb35b3918bbebff as hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/table/a90c7dafe9a548ca9fb35b3918bbebff 2024-11-08T12:44:29,729 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/table/a90c7dafe9a548ca9fb35b3918bbebff, entries=2, sequenceid=11, filesize=5.1 K 2024-11-08T12:44:29,731 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 675ms, sequenceid=11, compaction requested=false 2024-11-08T12:44:29,732 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T12:44:29,769 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-08T12:44:29,770 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T12:44:29,771 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T12:44:29,771 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731069869055Running coprocessor pre-close hooks at 1731069869055Disabling compacts and flushes for region at 1731069869055Disabling writes for close at 1731069869056 (+1 ms)Obtaining lock to block concurrent updates at 1731069869056Preparing flush snapshotting stores in 1588230740 at 1731069869056Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1377, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1731069869057 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731069869058 (+1 ms)Flushing 1588230740/info: creating writer at 1731069869058Flushing 1588230740/info: appending metadata at 1731069869095 (+37 ms)Flushing 1588230740/info: closing flushed file at 1731069869095Flushing 1588230740/ns: creating writer at 1731069869532 (+437 ms)Flushing 1588230740/ns: appending metadata at 1731069869549 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731069869550 (+1 ms)Flushing 1588230740/table: creating writer at 1731069869576 (+26 ms)Flushing 1588230740/table: appending metadata at 1731069869594 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731069869594Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@392596c6: reopening flushed file at 1731069869644 (+50 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f6c809: reopening flushed file at 1731069869661 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ea4c0a8: reopening flushed file at 1731069869706 (+45 ms)Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 675ms, sequenceid=11, compaction requested=false at 1731069869732 (+26 ms)Writing region close event to WAL at 1731069869736 (+4 ms)Running coprocessor post-close hooks at 1731069869770 (+34 ms)Closed at 1731069869771 (+1 ms) 2024-11-08T12:44:29,771 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T12:44:29,857 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(976): stopping server 7d22a5ea652e,37729,1731069864242; all regions closed. 2024-11-08T12:44:29,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741829_1019 (size=2751) 2024-11-08T12:44:29,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_1073741829_1019 (size=2751) 2024-11-08T12:44:29,866 DEBUG [RS:2;7d22a5ea652e:37729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/oldWALs 2024-11-08T12:44:29,866 INFO [RS:2;7d22a5ea652e:37729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d22a5ea652e%2C37729%2C1731069864242.meta:.meta(num 1731069866705) 2024-11-08T12:44:29,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_1073741827_1017 (size=93) 2024-11-08T12:44:29,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741827_1017 (size=93) 2024-11-08T12:44:29,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_1073741827_1017 (size=93) 2024-11-08T12:44:29,878 DEBUG [RS:2;7d22a5ea652e:37729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/oldWALs 2024-11-08T12:44:29,878 INFO [RS:2;7d22a5ea652e:37729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7d22a5ea652e%2C37729%2C1731069864242:(num 1731069866192) 2024-11-08T12:44:29,878 DEBUG [RS:2;7d22a5ea652e:37729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:29,878 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:29,878 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T12:44:29,878 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.ChoreService(370): Chore service for: regionserver/7d22a5ea652e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T12:44:29,878 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T12:44:29,878 INFO [regionserver/7d22a5ea652e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T12:44:29,879 INFO [RS:2;7d22a5ea652e:37729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37729 2024-11-08T12:44:29,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d22a5ea652e,37729,1731069864242 2024-11-08T12:44:29,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T12:44:29,881 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T12:44:29,883 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d22a5ea652e,37729,1731069864242] 2024-11-08T12:44:29,884 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d22a5ea652e,37729,1731069864242 already deleted, retry=false 2024-11-08T12:44:29,884 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d22a5ea652e,37729,1731069864242 expired; onlineServers=0 2024-11-08T12:44:29,884 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7d22a5ea652e,44263,1731069863207' ***** 2024-11-08T12:44:29,884 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T12:44:29,885 INFO [M:0;7d22a5ea652e:44263 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T12:44:29,885 INFO [M:0;7d22a5ea652e:44263 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T12:44:29,885 DEBUG [M:0;7d22a5ea652e:44263 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T12:44:29,885 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T12:44:29,885 DEBUG [M:0;7d22a5ea652e:44263 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T12:44:29,885 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.small.0-1731069865777 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.small.0-1731069865777,5,FailOnTimeoutGroup] 2024-11-08T12:44:29,885 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.large.0-1731069865767 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.large.0-1731069865767,5,FailOnTimeoutGroup] 2024-11-08T12:44:29,885 INFO [M:0;7d22a5ea652e:44263 {}] hbase.ChoreService(370): Chore service for: master/7d22a5ea652e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T12:44:29,885 INFO [M:0;7d22a5ea652e:44263 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T12:44:29,886 DEBUG [M:0;7d22a5ea652e:44263 {}] master.HMaster(1795): Stopping service threads 2024-11-08T12:44:29,886 INFO [M:0;7d22a5ea652e:44263 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T12:44:29,886 INFO [M:0;7d22a5ea652e:44263 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T12:44:29,886 INFO [M:0;7d22a5ea652e:44263 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T12:44:29,887 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T12:44:29,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T12:44:29,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:29,888 DEBUG [M:0;7d22a5ea652e:44263 {}] zookeeper.ZKUtil(347): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T12:44:29,888 WARN [M:0;7d22a5ea652e:44263 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T12:44:29,889 INFO [M:0;7d22a5ea652e:44263 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/.lastflushedseqids 2024-11-08T12:44:29,902 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,903 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,906 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:33070 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775584_1032] {}] datanode.DataXceiver(331): 127.0.0.1:40469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33070 dst: /127.0.0.1:40469 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:29,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_-9223372036854775584_1033 (size=127) 2024-11-08T12:44:29,910 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:29,911 INFO [M:0;7d22a5ea652e:44263 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T12:44:29,911 INFO [M:0;7d22a5ea652e:44263 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T12:44:29,911 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T12:44:29,911 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:29,911 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:29,911 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T12:44:29,911 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:29,911 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=26.84 KB heapSize=34.13 KB 2024-11-08T12:44:29,939 DEBUG [M:0;7d22a5ea652e:44263 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2683824761da4cd7bda6cf00988025ae is 82, key is hbase:meta,,1/info:regioninfo/1731069866829/Put/seqid=0 2024-11-08T12:44:29,942 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,942 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,949 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:45718 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775568_1034] {}] datanode.DataXceiver(331): 127.0.0.1:34381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45718 dst: /127.0.0.1:34381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:29,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_-9223372036854775568_1035 (size=5672) 2024-11-08T12:44:29,958 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:29,959 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2683824761da4cd7bda6cf00988025ae 2024-11-08T12:44:29,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:29,983 INFO [RS:2;7d22a5ea652e:37729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T12:44:29,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37729-0x1001f6bebaa0003, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:29,983 INFO [RS:2;7d22a5ea652e:37729 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d22a5ea652e,37729,1731069864242; zookeeper connection closed. 2024-11-08T12:44:29,984 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@396918d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@396918d 2024-11-08T12:44:29,984 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T12:44:29,991 DEBUG [M:0;7d22a5ea652e:44263 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57722efe150a4968987e7ed81e838af1 is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731069867734/Put/seqid=0 2024-11-08T12:44:29,994 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:29,994 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:30,001 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:33094 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775552_1036] {}] datanode.DataXceiver(331): 127.0.0.1:40469:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33094 dst: /127.0.0.1:40469 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:30,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_-9223372036854775552_1037 (size=6440) 2024-11-08T12:44:30,008 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:30,008 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.16 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57722efe150a4968987e7ed81e838af1 2024-11-08T12:44:30,036 DEBUG [M:0;7d22a5ea652e:44263 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39bb667b888f43e49e0fd4f4e8ac21f5 is 69, key is 7d22a5ea652e,36729,1731069864200/rs:state/1731069865848/Put/seqid=0 2024-11-08T12:44:30,039 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=3, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:30,039 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(531): Cannot allocate parity block(index=4, policy=RS-3-2-1024k). Exclude nodes=[]. There may not be enough datanodes or racks. You can check if the cluster topology supports the enabled erasure coding policies by running the command 'hdfs ec -verifyClusterSetup'. 2024-11-08T12:44:30,042 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1748978873_22 at /127.0.0.1:46004 [Receiving block BP-1671604421-172.17.0.3-1731069858349:blk_-9223372036854775536_1038] {}] datanode.DataXceiver(331): 127.0.0.1:39559:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46004 dst: /127.0.0.1:39559 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T12:44:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_-9223372036854775536_1039 (size=5294) 2024-11-08T12:44:30,052 WARN [M:0;7d22a5ea652e:44263 {}] hdfs.DFSStripedOutputStream(1367): Block group <1> failed to write 2 blocks. It's at high risk of losing data. 2024-11-08T12:44:30,052 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=195 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39bb667b888f43e49e0fd4f4e8ac21f5 2024-11-08T12:44:30,063 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2683824761da4cd7bda6cf00988025ae as hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2683824761da4cd7bda6cf00988025ae 2024-11-08T12:44:30,075 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2683824761da4cd7bda6cf00988025ae, entries=8, sequenceid=72, filesize=5.5 K 2024-11-08T12:44:30,077 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57722efe150a4968987e7ed81e838af1 as hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/57722efe150a4968987e7ed81e838af1 2024-11-08T12:44:30,086 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/57722efe150a4968987e7ed81e838af1, entries=8, sequenceid=72, filesize=6.3 K 2024-11-08T12:44:30,087 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39bb667b888f43e49e0fd4f4e8ac21f5 as hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/39bb667b888f43e49e0fd4f4e8ac21f5 2024-11-08T12:44:30,096 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/39bb667b888f43e49e0fd4f4e8ac21f5, entries=3, sequenceid=72, filesize=5.2 K 2024-11-08T12:44:30,097 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27483, heapSize ~33.83 KB/34640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 186ms, sequenceid=72, compaction requested=false 2024-11-08T12:44:30,099 INFO [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:30,099 DEBUG [M:0;7d22a5ea652e:44263 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731069869911Disabling compacts and flushes for region at 1731069869911Disabling writes for close at 1731069869911Obtaining lock to block concurrent updates at 1731069869911Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731069869911Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=27483, getHeapSize=34880, getOffHeapSize=0, getCellsCount=85 at 1731069869912 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731069869913 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731069869913Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731069869938 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731069869938Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731069869969 (+31 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731069869990 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731069869991 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731069870017 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731069870036 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731069870036Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c704d06: reopening flushed file at 1731069870061 (+25 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f3bbc2a: reopening flushed file at 1731069870075 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9b28280: reopening flushed file at 1731069870086 (+11 ms)Finished flush of dataSize ~26.84 KB/27483, heapSize ~33.83 KB/34640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 186ms, sequenceid=72, compaction requested=false at 1731069870097 (+11 ms)Writing region close event to WAL at 1731069870099 (+2 ms)Closed at 1731069870099 2024-11-08T12:44:30,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34381 is added to blk_1073741825_1011 (size=32686) 2024-11-08T12:44:30,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40469 is added to blk_1073741825_1011 (size=32686) 2024-11-08T12:44:30,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39559 is added to blk_1073741825_1011 (size=32686) 2024-11-08T12:44:30,105 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T12:44:30,105 INFO [M:0;7d22a5ea652e:44263 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T12:44:30,106 INFO [M:0;7d22a5ea652e:44263 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44263 2024-11-08T12:44:30,106 INFO [M:0;7d22a5ea652e:44263 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T12:44:30,208 INFO [M:0;7d22a5ea652e:44263 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T12:44:30,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:30,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44263-0x1001f6bebaa0000, quorum=127.0.0.1:55950, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:30,212 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54572a6d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:30,215 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54a3c380{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T12:44:30,215 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T12:44:30,215 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7353365d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T12:44:30,215 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@513d748e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir/,STOPPED} 2024-11-08T12:44:30,218 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T12:44:30,218 WARN [BP-1671604421-172.17.0.3-1731069858349 heartbeating to localhost/127.0.0.1:36053 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T12:44:30,218 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T12:44:30,218 WARN [BP-1671604421-172.17.0.3-1731069858349 heartbeating to localhost/127.0.0.1:36053 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1671604421-172.17.0.3-1731069858349 (Datanode Uuid 7f9f972a-bd45-47f8-9d9b-852f924a86b9) service to localhost/127.0.0.1:36053 2024-11-08T12:44:30,219 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data5/current/BP-1671604421-172.17.0.3-1731069858349 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:30,220 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data6/current/BP-1671604421-172.17.0.3-1731069858349 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:30,220 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T12:44:30,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3ea31b9a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:30,229 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@67374f99{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T12:44:30,229 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T12:44:30,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1663c233{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T12:44:30,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10ea181c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir/,STOPPED} 2024-11-08T12:44:30,231 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T12:44:30,231 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T12:44:30,232 WARN [BP-1671604421-172.17.0.3-1731069858349 heartbeating to localhost/127.0.0.1:36053 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T12:44:30,232 WARN [BP-1671604421-172.17.0.3-1731069858349 heartbeating to localhost/127.0.0.1:36053 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1671604421-172.17.0.3-1731069858349 (Datanode Uuid ca8f41a9-2645-4d4d-87ff-eaeee3311fc5) service to localhost/127.0.0.1:36053 2024-11-08T12:44:30,233 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data3/current/BP-1671604421-172.17.0.3-1731069858349 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:30,233 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data4/current/BP-1671604421-172.17.0.3-1731069858349 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:30,234 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T12:44:30,238 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@321ada2c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:30,238 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@637479e9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T12:44:30,238 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T12:44:30,239 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5f7daeb1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T12:44:30,239 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51c14b1c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir/,STOPPED} 2024-11-08T12:44:30,241 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T12:44:30,241 WARN [BP-1671604421-172.17.0.3-1731069858349 heartbeating to localhost/127.0.0.1:36053 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T12:44:30,241 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T12:44:30,241 WARN [BP-1671604421-172.17.0.3-1731069858349 heartbeating to localhost/127.0.0.1:36053 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1671604421-172.17.0.3-1731069858349 (Datanode Uuid 2a614efd-e11c-454c-9df3-621e978018cc) service to localhost/127.0.0.1:36053 2024-11-08T12:44:30,241 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data1/current/BP-1671604421-172.17.0.3-1731069858349 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:30,241 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/cluster_71d88abf-c373-327d-a9a3-53fbb587d2f5/data/data2/current/BP-1671604421-172.17.0.3-1731069858349 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:30,242 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T12:44:30,259 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@76e22261{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T12:44:30,261 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3599471c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T12:44:30,261 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T12:44:30,261 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2faf2775{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T12:44:30,262 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e18bd18{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir/,STOPPED} 2024-11-08T12:44:30,279 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T12:44:30,312 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T12:44:30,321 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestHBaseWalOnEC#testReadWrite[0] Thread=88 (was 157), OpenFileDescriptor=435 (was 391) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=454 (was 379) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1689 (was 2747) 2024-11-08T12:44:30,328 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestHBaseWalOnEC#testReadWrite[1] Thread=88, OpenFileDescriptor=435, MaxFileDescriptor=1048576, SystemLoadAverage=454, ProcessCount=11, AvailableMemoryMB=1688 2024-11-08T12:44:30,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T12:44:30,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.log.dir so I do NOT create it in target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c 2024-11-08T12:44:30,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d289399e-76f6-8cb8-16f1-e21f5e4a42c4/hadoop.tmp.dir so I do NOT create it in target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c 2024-11-08T12:44:30,329 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2, deleteOnExit=true 2024-11-08T12:44:30,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T12:44:30,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/test.cache.data in system properties and HBase conf 2024-11-08T12:44:30,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T12:44:30,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir in system properties and HBase conf 2024-11-08T12:44:30,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T12:44:30,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T12:44:30,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T12:44:30,330 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T12:44:30,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T12:44:30,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T12:44:30,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T12:44:30,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T12:44:30,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T12:44:30,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T12:44:30,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T12:44:30,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T12:44:30,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T12:44:30,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/nfs.dump.dir in system properties and HBase conf 2024-11-08T12:44:30,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/java.io.tmpdir in system properties and HBase conf 2024-11-08T12:44:30,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T12:44:30,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T12:44:30,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T12:44:30,428 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:30,434 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T12:44:30,435 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T12:44:30,435 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T12:44:30,436 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T12:44:30,437 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:30,440 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18bcac98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir/,AVAILABLE} 2024-11-08T12:44:30,441 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68969217{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T12:44:30,568 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4b64f5ed{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/java.io.tmpdir/jetty-localhost-45817-hadoop-hdfs-3_4_1-tests_jar-_-any-4119209080420655522/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T12:44:30,569 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c00a7dd{HTTP/1.1, (http/1.1)}{localhost:45817} 2024-11-08T12:44:30,569 INFO [Time-limited test {}] server.Server(415): Started @15268ms 2024-11-08T12:44:30,665 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:30,668 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T12:44:30,669 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T12:44:30,669 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T12:44:30,669 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T12:44:30,669 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33ecba79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir/,AVAILABLE} 2024-11-08T12:44:30,670 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54931008{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T12:44:30,791 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7005d2e9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/java.io.tmpdir/jetty-localhost-33049-hadoop-hdfs-3_4_1-tests_jar-_-any-12905527113261278355/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:30,791 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6496cd5{HTTP/1.1, (http/1.1)}{localhost:33049} 2024-11-08T12:44:30,792 INFO [Time-limited test {}] server.Server(415): Started @15491ms 2024-11-08T12:44:30,793 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T12:44:30,834 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:30,837 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T12:44:30,839 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T12:44:30,839 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T12:44:30,839 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T12:44:30,840 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3dc09a89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir/,AVAILABLE} 2024-11-08T12:44:30,840 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b034d27{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T12:44:30,905 WARN [Thread-553 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data1/current/BP-549379526-172.17.0.3-1731069870368/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:30,905 WARN [Thread-554 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data2/current/BP-549379526-172.17.0.3-1731069870368/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:30,950 WARN [Thread-532 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T12:44:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1590864c0987c7c9 with lease ID 0xa29fce053761758c: Processing first storage report for DS-6ba168d1-7c4f-4532-8467-20bd099832f7 from datanode DatanodeRegistration(127.0.0.1:42371, datanodeUuid=bf3ca83b-feac-45ef-ba4e-b96c843104db, infoPort=39195, infoSecurePort=0, ipcPort=41931, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368) 2024-11-08T12:44:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1590864c0987c7c9 with lease ID 0xa29fce053761758c: from storage DS-6ba168d1-7c4f-4532-8467-20bd099832f7 node DatanodeRegistration(127.0.0.1:42371, datanodeUuid=bf3ca83b-feac-45ef-ba4e-b96c843104db, infoPort=39195, infoSecurePort=0, ipcPort=41931, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T12:44:30,954 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1590864c0987c7c9 with lease ID 0xa29fce053761758c: Processing first storage report for DS-4f8adfa0-b2bb-4e54-8931-d1121a04c2a3 from datanode DatanodeRegistration(127.0.0.1:42371, datanodeUuid=bf3ca83b-feac-45ef-ba4e-b96c843104db, infoPort=39195, infoSecurePort=0, ipcPort=41931, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368) 2024-11-08T12:44:30,954 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1590864c0987c7c9 with lease ID 0xa29fce053761758c: from storage DS-4f8adfa0-b2bb-4e54-8931-d1121a04c2a3 node DatanodeRegistration(127.0.0.1:42371, datanodeUuid=bf3ca83b-feac-45ef-ba4e-b96c843104db, infoPort=39195, infoSecurePort=0, ipcPort=41931, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T12:44:30,969 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26cf3f54{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/java.io.tmpdir/jetty-localhost-44531-hadoop-hdfs-3_4_1-tests_jar-_-any-10893640152998595935/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:30,969 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10498144{HTTP/1.1, (http/1.1)}{localhost:44531} 2024-11-08T12:44:30,970 INFO [Time-limited test {}] server.Server(415): Started @15669ms 2024-11-08T12:44:30,971 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T12:44:31,006 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T12:44:31,010 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T12:44:31,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T12:44:31,010 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T12:44:31,010 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T12:44:31,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@638f7c01{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir/,AVAILABLE} 2024-11-08T12:44:31,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41154aac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T12:44:31,074 WARN [Thread-588 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data3/current/BP-549379526-172.17.0.3-1731069870368/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:31,074 WARN [Thread-589 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data4/current/BP-549379526-172.17.0.3-1731069870368/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:31,097 WARN [Thread-568 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T12:44:31,104 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x270f53de86139ff7 with lease ID 0xa29fce053761758d: Processing first storage report for DS-1eb7ccd1-acb2-4ae3-8a61-e69cf6fafa31 from datanode DatanodeRegistration(127.0.0.1:35653, datanodeUuid=70cc5c95-972b-4c66-8c75-887e1111027e, infoPort=46159, infoSecurePort=0, ipcPort=35129, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368) 2024-11-08T12:44:31,104 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x270f53de86139ff7 with lease ID 0xa29fce053761758d: from storage DS-1eb7ccd1-acb2-4ae3-8a61-e69cf6fafa31 node DatanodeRegistration(127.0.0.1:35653, datanodeUuid=70cc5c95-972b-4c66-8c75-887e1111027e, infoPort=46159, infoSecurePort=0, ipcPort=35129, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T12:44:31,104 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x270f53de86139ff7 with lease ID 0xa29fce053761758d: Processing first storage report for DS-734db84b-8bc7-4f2c-8552-7bbf5d108390 from datanode DatanodeRegistration(127.0.0.1:35653, datanodeUuid=70cc5c95-972b-4c66-8c75-887e1111027e, infoPort=46159, infoSecurePort=0, ipcPort=35129, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368) 2024-11-08T12:44:31,104 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x270f53de86139ff7 with lease ID 0xa29fce053761758d: from storage DS-734db84b-8bc7-4f2c-8552-7bbf5d108390 node DatanodeRegistration(127.0.0.1:35653, datanodeUuid=70cc5c95-972b-4c66-8c75-887e1111027e, infoPort=46159, infoSecurePort=0, ipcPort=35129, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T12:44:31,135 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42279afb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/java.io.tmpdir/jetty-localhost-44087-hadoop-hdfs-3_4_1-tests_jar-_-any-17439492969507784237/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:31,136 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776d8022{HTTP/1.1, (http/1.1)}{localhost:44087} 2024-11-08T12:44:31,136 INFO [Time-limited test {}] server.Server(415): Started @15835ms 2024-11-08T12:44:31,137 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T12:44:31,245 WARN [Thread-614 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data5/current/BP-549379526-172.17.0.3-1731069870368/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:31,249 WARN [Thread-615 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data6/current/BP-549379526-172.17.0.3-1731069870368/current, will proceed with Du for space computation calculation, 2024-11-08T12:44:31,270 WARN [Thread-603 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T12:44:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd5349eea7bcd41a0 with lease ID 0xa29fce053761758e: Processing first storage report for DS-f7f60235-3ffd-43a3-b3aa-9be38dd56b30 from datanode DatanodeRegistration(127.0.0.1:42227, datanodeUuid=44007310-83ab-4f61-a09c-a75c8f0e1a3e, infoPort=42663, infoSecurePort=0, ipcPort=34313, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368) 2024-11-08T12:44:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd5349eea7bcd41a0 with lease ID 0xa29fce053761758e: from storage DS-f7f60235-3ffd-43a3-b3aa-9be38dd56b30 node DatanodeRegistration(127.0.0.1:42227, datanodeUuid=44007310-83ab-4f61-a09c-a75c8f0e1a3e, infoPort=42663, infoSecurePort=0, ipcPort=34313, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T12:44:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd5349eea7bcd41a0 with lease ID 0xa29fce053761758e: Processing first storage report for DS-9f7f6dc7-3b57-4f02-bc23-a6fe92bcffa4 from datanode DatanodeRegistration(127.0.0.1:42227, datanodeUuid=44007310-83ab-4f61-a09c-a75c8f0e1a3e, infoPort=42663, infoSecurePort=0, ipcPort=34313, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368) 2024-11-08T12:44:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd5349eea7bcd41a0 with lease ID 0xa29fce053761758e: from storage DS-9f7f6dc7-3b57-4f02-bc23-a6fe92bcffa4 node DatanodeRegistration(127.0.0.1:42227, datanodeUuid=44007310-83ab-4f61-a09c-a75c8f0e1a3e, infoPort=42663, infoSecurePort=0, ipcPort=34313, storageInfo=lv=-57;cid=testClusterID;nsid=381283891;c=1731069870368), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T12:44:31,370 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c 2024-11-08T12:44:31,373 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/zookeeper_0, clientPort=51233, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T12:44:31,374 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51233 2024-11-08T12:44:31,374 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,376 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741825_1001 (size=7) 2024-11-08T12:44:31,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741825_1001 (size=7) 2024-11-08T12:44:31,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741825_1001 (size=7) 2024-11-08T12:44:31,392 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3 with version=8 2024-11-08T12:44:31,392 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36053/user/jenkins/test-data/6e0f4611-5762-a2cc-83c5-b163d9d7a63e/hbase-staging 2024-11-08T12:44:31,394 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7d22a5ea652e:0 server-side Connection retries=45 2024-11-08T12:44:31,394 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,394 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,394 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T12:44:31,394 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,394 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T12:44:31,395 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T12:44:31,395 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T12:44:31,395 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46881 2024-11-08T12:44:31,398 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46881 connecting to ZooKeeper ensemble=127.0.0.1:51233 2024-11-08T12:44:31,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:468810x0, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T12:44:31,405 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46881-0x1001f6c0faa0000 connected 2024-11-08T12:44:31,421 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,423 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:31,425 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3, hbase.cluster.distributed=false 2024-11-08T12:44:31,427 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T12:44:31,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46881 2024-11-08T12:44:31,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46881 2024-11-08T12:44:31,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46881 2024-11-08T12:44:31,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46881 2024-11-08T12:44:31,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46881 2024-11-08T12:44:31,447 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d22a5ea652e:0 server-side Connection retries=45 2024-11-08T12:44:31,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,448 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T12:44:31,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,448 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T12:44:31,448 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T12:44:31,448 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T12:44:31,449 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45869 2024-11-08T12:44:31,450 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45869 connecting to ZooKeeper ensemble=127.0.0.1:51233 2024-11-08T12:44:31,451 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,453 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458690x0, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T12:44:31,461 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45869-0x1001f6c0faa0001 connected 2024-11-08T12:44:31,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:31,462 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T12:44:31,463 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T12:44:31,464 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T12:44:31,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T12:44:31,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45869 2024-11-08T12:44:31,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45869 2024-11-08T12:44:31,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45869 2024-11-08T12:44:31,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45869 2024-11-08T12:44:31,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45869 2024-11-08T12:44:31,487 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d22a5ea652e:0 server-side Connection retries=45 2024-11-08T12:44:31,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,487 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T12:44:31,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T12:44:31,487 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T12:44:31,488 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T12:44:31,488 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41661 2024-11-08T12:44:31,490 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41661 connecting to ZooKeeper ensemble=127.0.0.1:51233 2024-11-08T12:44:31,491 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,493 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416610x0, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T12:44:31,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:416610x0, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:31,499 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41661-0x1001f6c0faa0002 connected 2024-11-08T12:44:31,499 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T12:44:31,504 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T12:44:31,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T12:44:31,507 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T12:44:31,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41661 2024-11-08T12:44:31,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41661 2024-11-08T12:44:31,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41661 2024-11-08T12:44:31,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41661 2024-11-08T12:44:31,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41661 2024-11-08T12:44:31,547 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7d22a5ea652e:0 server-side Connection retries=45 2024-11-08T12:44:31,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,548 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T12:44:31,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T12:44:31,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T12:44:31,548 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T12:44:31,548 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T12:44:31,552 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46679 2024-11-08T12:44:31,553 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46679 connecting to ZooKeeper ensemble=127.0.0.1:51233 2024-11-08T12:44:31,554 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,556 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466790x0, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T12:44:31,561 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:466790x0, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:31,561 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46679-0x1001f6c0faa0003 connected 2024-11-08T12:44:31,562 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T12:44:31,565 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T12:44:31,565 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T12:44:31,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T12:44:31,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46679 2024-11-08T12:44:31,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46679 2024-11-08T12:44:31,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46679 2024-11-08T12:44:31,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46679 2024-11-08T12:44:31,569 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46679 2024-11-08T12:44:31,584 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7d22a5ea652e:46881 2024-11-08T12:44:31,584 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:31,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:31,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:31,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:31,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:31,593 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:31,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T12:44:31,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T12:44:31,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T12:44:31,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,599 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T12:44:31,599 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7d22a5ea652e,46881,1731069871394 from backup master directory 2024-11-08T12:44:31,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:31,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:31,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:31,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:31,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T12:44:31,607 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T12:44:31,608 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:31,619 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/hbase.id] with ID: 1a9516c5-c300-4b5d-ba4c-437b6c2b9a44 2024-11-08T12:44:31,619 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/.tmp/hbase.id 2024-11-08T12:44:31,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741826_1002 (size=42) 2024-11-08T12:44:31,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741826_1002 (size=42) 2024-11-08T12:44:31,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741826_1002 (size=42) 2024-11-08T12:44:31,644 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/.tmp/hbase.id]:[hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/hbase.id] 2024-11-08T12:44:31,674 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T12:44:31,674 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T12:44:31,676 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-08T12:44:31,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741827_1003 (size=196) 2024-11-08T12:44:31,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741827_1003 (size=196) 2024-11-08T12:44:31,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741827_1003 (size=196) 2024-11-08T12:44:31,702 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T12:44:31,703 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T12:44:31,704 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-08T12:44:31,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741828_1004 (size=1189) 2024-11-08T12:44:31,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741828_1004 (size=1189) 2024-11-08T12:44:31,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741828_1004 (size=1189) 2024-11-08T12:44:31,746 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store 2024-11-08T12:44:31,755 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-08T12:44:31,756 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-08T12:44:31,757 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-08T12:44:31,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741829_1005 (size=34) 2024-11-08T12:44:31,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741829_1005 (size=34) 2024-11-08T12:44:31,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741829_1005 (size=34) 2024-11-08T12:44:31,774 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:31,774 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T12:44:31,774 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:31,774 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:31,775 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T12:44:31,775 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:31,775 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:31,775 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731069871774Disabling compacts and flushes for region at 1731069871774Disabling writes for close at 1731069871775 (+1 ms)Writing region close event to WAL at 1731069871775Closed at 1731069871775 2024-11-08T12:44:31,776 WARN [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/.initializing 2024-11-08T12:44:31,776 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/WALs/7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:31,785 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C46881%2C1731069871394, suffix=, logDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/WALs/7d22a5ea652e,46881,1731069871394, archiveDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/oldWALs, maxLogs=10 2024-11-08T12:44:31,786 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7d22a5ea652e%2C46881%2C1731069871394.1731069871786 2024-11-08T12:44:31,804 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/WALs/7d22a5ea652e,46881,1731069871394/7d22a5ea652e%2C46881%2C1731069871394.1731069871786 2024-11-08T12:44:31,807 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39195:39195),(127.0.0.1/127.0.0.1:42663:42663),(127.0.0.1/127.0.0.1:46159:46159)] 2024-11-08T12:44:31,808 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T12:44:31,809 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:31,809 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,809 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T12:44:31,813 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:31,814 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T12:44:31,816 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T12:44:31,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T12:44:31,820 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T12:44:31,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T12:44:31,823 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T12:44:31,824 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,824 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,825 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,827 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,827 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,828 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T12:44:31,830 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T12:44:31,837 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T12:44:31,838 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62603070, jitterRate=-0.0671415627002716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T12:44:31,839 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731069871809Initializing all the Stores at 1731069871811 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069871811Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069871811Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069871811Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069871811Cleaning up temporary data from old regions at 1731069871827 (+16 ms)Region opened successfully at 1731069871839 (+12 ms) 2024-11-08T12:44:31,840 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T12:44:31,845 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eff199e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d22a5ea652e/172.17.0.3:0 2024-11-08T12:44:31,847 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T12:44:31,847 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T12:44:31,847 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T12:44:31,847 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T12:44:31,848 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T12:44:31,849 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-08T12:44:31,849 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T12:44:31,854 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T12:44:31,855 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T12:44:31,857 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T12:44:31,857 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T12:44:31,858 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T12:44:31,859 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T12:44:31,859 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T12:44:31,861 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T12:44:31,863 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T12:44:31,864 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T12:44:31,865 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T12:44:31,868 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T12:44:31,870 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T12:44:31,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:31,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:31,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:31,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:31,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,874 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7d22a5ea652e,46881,1731069871394, sessionid=0x1001f6c0faa0000, setting cluster-up flag (Was=false) 2024-11-08T12:44:31,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,885 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T12:44:31,887 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:31,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:31,896 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T12:44:31,897 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:31,899 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T12:44:31,901 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T12:44:31,902 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T12:44:31,902 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T12:44:31,902 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7d22a5ea652e,46881,1731069871394 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T12:44:31,904 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7d22a5ea652e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T12:44:31,904 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7d22a5ea652e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T12:44:31,904 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T12:44:31,905 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=5, maxPoolSize=5 2024-11-08T12:44:31,905 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7d22a5ea652e:0, corePoolSize=10, maxPoolSize=10 2024-11-08T12:44:31,905 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:31,905 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T12:44:31,905 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:31,906 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731069901906 2024-11-08T12:44:31,906 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T12:44:31,906 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T12:44:31,906 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T12:44:31,906 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T12:44:31,906 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T12:44:31,906 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T12:44:31,907 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:31,907 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T12:44:31,907 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T12:44:31,907 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T12:44:31,907 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T12:44:31,908 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T12:44:31,908 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T12:44:31,908 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T12:44:31,908 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.large.0-1731069871908,5,FailOnTimeoutGroup] 2024-11-08T12:44:31,908 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.small.0-1731069871908,5,FailOnTimeoutGroup] 2024-11-08T12:44:31,908 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:31,908 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T12:44:31,909 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:31,909 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:31,909 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,909 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T12:44:31,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741831_1007 (size=1321) 2024-11-08T12:44:31,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741831_1007 (size=1321) 2024-11-08T12:44:31,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741831_1007 (size=1321) 2024-11-08T12:44:31,923 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T12:44:31,923 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3 2024-11-08T12:44:31,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741832_1008 (size=32) 2024-11-08T12:44:31,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741832_1008 (size=32) 2024-11-08T12:44:31,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741832_1008 (size=32) 2024-11-08T12:44:31,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:31,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T12:44:31,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T12:44:31,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:31,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T12:44:31,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T12:44:31,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:31,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T12:44:31,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T12:44:31,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:31,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T12:44:31,955 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T12:44:31,955 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:31,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:31,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T12:44:31,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740 2024-11-08T12:44:31,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740 2024-11-08T12:44:31,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T12:44:31,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T12:44:31,967 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T12:44:31,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T12:44:31,974 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(746): ClusterId : 1a9516c5-c300-4b5d-ba4c-437b6c2b9a44 2024-11-08T12:44:31,974 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T12:44:31,977 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T12:44:31,978 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T12:44:31,978 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T12:44:31,978 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74799063, jitterRate=0.11459289491176605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T12:44:31,979 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(746): ClusterId : 1a9516c5-c300-4b5d-ba4c-437b6c2b9a44 2024-11-08T12:44:31,979 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T12:44:31,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731069871939Initializing all the Stores at 1731069871941 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069871941Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069871942 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069871942Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069871942Cleaning up temporary data from old regions at 1731069871966 (+24 ms)Region opened successfully at 1731069871979 (+13 ms) 2024-11-08T12:44:31,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T12:44:31,980 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T12:44:31,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T12:44:31,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T12:44:31,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T12:44:31,981 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(746): ClusterId : 1a9516c5-c300-4b5d-ba4c-437b6c2b9a44 2024-11-08T12:44:31,981 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T12:44:31,981 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T12:44:31,982 DEBUG [RS:0;7d22a5ea652e:45869 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acf5b10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d22a5ea652e/172.17.0.3:0 2024-11-08T12:44:31,983 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T12:44:31,983 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T12:44:31,985 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T12:44:31,985 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T12:44:31,986 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T12:44:31,987 DEBUG [RS:1;7d22a5ea652e:41661 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbf3607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d22a5ea652e/172.17.0.3:0 2024-11-08T12:44:31,987 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T12:44:31,987 DEBUG [RS:2;7d22a5ea652e:46679 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18234df1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7d22a5ea652e/172.17.0.3:0 2024-11-08T12:44:31,989 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T12:44:31,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731069871980Disabling compacts and flushes for region at 1731069871980Disabling writes for close at 1731069871980Writing region close event to WAL at 1731069871989 (+9 ms)Closed at 1731069871989 2024-11-08T12:44:32,004 DEBUG [RS:0;7d22a5ea652e:45869 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7d22a5ea652e:45869 2024-11-08T12:44:32,004 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T12:44:32,005 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T12:44:32,005 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T12:44:32,005 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T12:44:32,005 DEBUG [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T12:44:32,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T12:44:32,008 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,46881,1731069871394 with port=45869, startcode=1731069871447 2024-11-08T12:44:32,008 DEBUG [RS:0;7d22a5ea652e:45869 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T12:44:32,009 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T12:44:32,011 DEBUG [RS:2;7d22a5ea652e:46679 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;7d22a5ea652e:46679 2024-11-08T12:44:32,012 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T12:44:32,012 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T12:44:32,012 DEBUG [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T12:44:32,013 DEBUG [RS:1;7d22a5ea652e:41661 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7d22a5ea652e:41661 2024-11-08T12:44:32,013 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T12:44:32,013 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T12:44:32,013 DEBUG [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T12:44:32,013 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,46881,1731069871394 with port=46679, startcode=1731069871547 2024-11-08T12:44:32,014 DEBUG [RS:2;7d22a5ea652e:46679 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T12:44:32,014 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(2659): reportForDuty to master=7d22a5ea652e,46881,1731069871394 with port=41661, startcode=1731069871486 2024-11-08T12:44:32,014 DEBUG [RS:1;7d22a5ea652e:41661 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T12:44:32,016 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T12:44:32,018 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39103, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T12:44:32,018 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34795, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T12:44:32,019 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46881 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:32,019 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46881 {}] master.ServerManager(517): Registering regionserver=7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:32,019 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47135, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T12:44:32,021 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46881 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d22a5ea652e,46679,1731069871547 2024-11-08T12:44:32,021 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46881 {}] master.ServerManager(517): Registering regionserver=7d22a5ea652e,46679,1731069871547 2024-11-08T12:44:32,021 DEBUG [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3 2024-11-08T12:44:32,021 DEBUG [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39575 2024-11-08T12:44:32,021 DEBUG [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T12:44:32,023 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46881 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,023 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46881 {}] master.ServerManager(517): Registering regionserver=7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,023 DEBUG [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3 2024-11-08T12:44:32,023 DEBUG [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39575 2024-11-08T12:44:32,023 DEBUG [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T12:44:32,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T12:44:32,026 DEBUG [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3 2024-11-08T12:44:32,026 DEBUG [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39575 2024-11-08T12:44:32,026 DEBUG [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T12:44:32,029 DEBUG [RS:0;7d22a5ea652e:45869 {}] zookeeper.ZKUtil(111): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:32,029 WARN [RS:0;7d22a5ea652e:45869 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T12:44:32,030 INFO [RS:0;7d22a5ea652e:45869 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-08T12:44:32,030 DEBUG [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:32,030 DEBUG [RS:2;7d22a5ea652e:46679 {}] zookeeper.ZKUtil(111): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d22a5ea652e,46679,1731069871547 2024-11-08T12:44:32,030 WARN [RS:2;7d22a5ea652e:46679 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T12:44:32,030 INFO [RS:2;7d22a5ea652e:46679 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-08T12:44:32,031 DEBUG [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,46679,1731069871547 2024-11-08T12:44:32,031 DEBUG [RS:1;7d22a5ea652e:41661 {}] zookeeper.ZKUtil(111): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,031 WARN [RS:1;7d22a5ea652e:41661 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T12:44:32,031 INFO [RS:1;7d22a5ea652e:41661 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-08T12:44:32,031 DEBUG [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,033 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d22a5ea652e,41661,1731069871486] 2024-11-08T12:44:32,033 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d22a5ea652e,46679,1731069871547] 2024-11-08T12:44:32,033 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7d22a5ea652e,45869,1731069871447] 2024-11-08T12:44:32,044 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T12:44:32,048 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T12:44:32,049 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T12:44:32,050 INFO [RS:0;7d22a5ea652e:45869 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T12:44:32,051 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,052 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T12:44:32,052 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T12:44:32,054 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T12:44:32,054 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d22a5ea652e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,055 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,056 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,056 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,056 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,056 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:32,056 DEBUG [RS:0;7d22a5ea652e:45869 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:32,060 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T12:44:32,060 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T12:44:32,060 INFO [RS:2;7d22a5ea652e:46679 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T12:44:32,061 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,064 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T12:44:32,064 INFO [RS:1;7d22a5ea652e:41661 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T12:44:32,064 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,064 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,064 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,064 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,064 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,064 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,064 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,45869,1731069871447-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T12:44:32,064 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T12:44:32,065 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T12:44:32,065 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,065 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,065 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T12:44:32,065 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,065 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,065 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d22a5ea652e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T12:44:32,066 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7d22a5ea652e:0, corePoolSize=2, maxPoolSize=2 2024-11-08T12:44:32,066 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,066 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,067 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,067 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,067 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:32,067 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,067 DEBUG [RS:2;7d22a5ea652e:46679 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:32,067 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,067 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7d22a5ea652e:0, corePoolSize=1, maxPoolSize=1 2024-11-08T12:44:32,067 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:32,067 DEBUG [RS:1;7d22a5ea652e:41661 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0, corePoolSize=3, maxPoolSize=3 2024-11-08T12:44:32,068 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,068 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,068 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,068 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,068 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,068 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,46679,1731069871547-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T12:44:32,070 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,070 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,070 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,070 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,070 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,070 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,41661,1731069871486-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T12:44:32,087 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T12:44:32,089 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,45869,1731069871447-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,089 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,089 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.Replication(171): 7d22a5ea652e,45869,1731069871447 started 2024-11-08T12:44:32,090 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T12:44:32,090 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,41661,1731069871486-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,090 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,090 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.Replication(171): 7d22a5ea652e,41661,1731069871486 started 2024-11-08T12:44:32,092 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T12:44:32,092 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,46679,1731069871547-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,092 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,093 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.Replication(171): 7d22a5ea652e,46679,1731069871547 started 2024-11-08T12:44:32,108 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,108 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(1482): Serving as 7d22a5ea652e,46679,1731069871547, RpcServer on 7d22a5ea652e/172.17.0.3:46679, sessionid=0x1001f6c0faa0003 2024-11-08T12:44:32,108 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T12:44:32,108 DEBUG [RS:2;7d22a5ea652e:46679 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d22a5ea652e,46679,1731069871547 2024-11-08T12:44:32,108 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,46679,1731069871547' 2024-11-08T12:44:32,108 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T12:44:32,109 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T12:44:32,110 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T12:44:32,110 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T12:44:32,110 DEBUG [RS:2;7d22a5ea652e:46679 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d22a5ea652e,46679,1731069871547 2024-11-08T12:44:32,110 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,46679,1731069871547' 2024-11-08T12:44:32,110 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T12:44:32,111 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T12:44:32,111 DEBUG [RS:2;7d22a5ea652e:46679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T12:44:32,111 INFO [RS:2;7d22a5ea652e:46679 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T12:44:32,111 INFO [RS:2;7d22a5ea652e:46679 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T12:44:32,111 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,111 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1482): Serving as 7d22a5ea652e,45869,1731069871447, RpcServer on 7d22a5ea652e/172.17.0.3:45869, sessionid=0x1001f6c0faa0001 2024-11-08T12:44:32,111 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T12:44:32,112 DEBUG [RS:0;7d22a5ea652e:45869 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:32,112 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,45869,1731069871447' 2024-11-08T12:44:32,112 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T12:44:32,112 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T12:44:32,112 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,113 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1482): Serving as 7d22a5ea652e,41661,1731069871486, RpcServer on 7d22a5ea652e/172.17.0.3:41661, sessionid=0x1001f6c0faa0002 2024-11-08T12:44:32,113 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T12:44:32,113 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T12:44:32,113 DEBUG [RS:0;7d22a5ea652e:45869 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:32,113 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,45869,1731069871447' 2024-11-08T12:44:32,113 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T12:44:32,113 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T12:44:32,113 DEBUG [RS:1;7d22a5ea652e:41661 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,113 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,41661,1731069871486' 2024-11-08T12:44:32,113 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T12:44:32,113 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T12:44:32,114 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T12:44:32,114 DEBUG [RS:0;7d22a5ea652e:45869 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T12:44:32,114 INFO [RS:0;7d22a5ea652e:45869 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T12:44:32,114 INFO [RS:0;7d22a5ea652e:45869 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T12:44:32,115 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T12:44:32,115 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T12:44:32,115 DEBUG [RS:1;7d22a5ea652e:41661 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,115 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7d22a5ea652e,41661,1731069871486' 2024-11-08T12:44:32,115 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T12:44:32,115 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T12:44:32,116 DEBUG [RS:1;7d22a5ea652e:41661 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T12:44:32,116 INFO [RS:1;7d22a5ea652e:41661 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T12:44:32,116 INFO [RS:1;7d22a5ea652e:41661 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T12:44:32,166 WARN [7d22a5ea652e:46881 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T12:44:32,216 INFO [RS:2;7d22a5ea652e:46679 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C46679%2C1731069871547, suffix=, logDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,46679,1731069871547, archiveDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/oldWALs, maxLogs=32 2024-11-08T12:44:32,219 INFO [RS:2;7d22a5ea652e:46679 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7d22a5ea652e%2C46679%2C1731069871547.1731069872219 2024-11-08T12:44:32,220 INFO [RS:1;7d22a5ea652e:41661 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C41661%2C1731069871486, suffix=, logDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,41661,1731069871486, archiveDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/oldWALs, maxLogs=32 2024-11-08T12:44:32,220 INFO [RS:0;7d22a5ea652e:45869 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C45869%2C1731069871447, suffix=, logDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,45869,1731069871447, archiveDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/oldWALs, maxLogs=32 2024-11-08T12:44:32,223 INFO [RS:1;7d22a5ea652e:41661 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7d22a5ea652e%2C41661%2C1731069871486.1731069872223 2024-11-08T12:44:32,225 INFO [RS:0;7d22a5ea652e:45869 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7d22a5ea652e%2C45869%2C1731069871447.1731069872225 2024-11-08T12:44:32,260 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-08T12:44:32,262 INFO [RS:1;7d22a5ea652e:41661 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,41661,1731069871486/7d22a5ea652e%2C41661%2C1731069871486.1731069872223 2024-11-08T12:44:32,264 INFO [RS:0;7d22a5ea652e:45869 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,45869,1731069871447/7d22a5ea652e%2C45869%2C1731069871447.1731069872225 2024-11-08T12:44:32,270 INFO [RS:2;7d22a5ea652e:46679 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,46679,1731069871547/7d22a5ea652e%2C46679%2C1731069871547.1731069872219 2024-11-08T12:44:32,273 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-08T12:44:32,279 DEBUG [RS:0;7d22a5ea652e:45869 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39195:39195),(127.0.0.1/127.0.0.1:42663:42663),(127.0.0.1/127.0.0.1:46159:46159)] 2024-11-08T12:44:32,285 DEBUG [RS:2;7d22a5ea652e:46679 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39195:39195),(127.0.0.1/127.0.0.1:42663:42663),(127.0.0.1/127.0.0.1:46159:46159)] 2024-11-08T12:44:32,296 DEBUG [RS:1;7d22a5ea652e:41661 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39195:39195),(127.0.0.1/127.0.0.1:46159:46159),(127.0.0.1/127.0.0.1:42663:42663)] 2024-11-08T12:44:32,328 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-08T12:44:32,328 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-08T12:44:32,417 DEBUG [7d22a5ea652e:46881 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-08T12:44:32,417 DEBUG [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(204): Hosts are {7d22a5ea652e=0} racks are {/default-rack=0} 2024-11-08T12:44:32,420 DEBUG [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T12:44:32,420 DEBUG [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T12:44:32,420 DEBUG [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T12:44:32,420 DEBUG [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T12:44:32,420 DEBUG [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T12:44:32,420 DEBUG [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T12:44:32,420 INFO [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T12:44:32,420 INFO [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T12:44:32,420 INFO [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T12:44:32,420 DEBUG [7d22a5ea652e:46881 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T12:44:32,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,425 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d22a5ea652e,41661,1731069871486, state=OPENING 2024-11-08T12:44:32,427 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T12:44:32,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:32,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:32,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:32,429 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:32,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:32,431 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T12:44:32,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7d22a5ea652e,41661,1731069871486}] 2024-11-08T12:44:32,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:32,434 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:32,437 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:32,594 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T12:44:32,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37203, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T12:44:32,602 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T12:44:32,603 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-08T12:44:32,605 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7d22a5ea652e%2C41661%2C1731069871486.meta, suffix=.meta, logDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,41661,1731069871486, archiveDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/oldWALs, maxLogs=32 2024-11-08T12:44:32,606 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7d22a5ea652e%2C41661%2C1731069871486.meta.1731069872606.meta 2024-11-08T12:44:32,616 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/WALs/7d22a5ea652e,41661,1731069871486/7d22a5ea652e%2C41661%2C1731069871486.meta.1731069872606.meta 2024-11-08T12:44:32,621 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39195:39195),(127.0.0.1/127.0.0.1:42663:42663),(127.0.0.1/127.0.0.1:46159:46159)] 2024-11-08T12:44:32,625 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T12:44:32,626 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T12:44:32,626 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T12:44:32,626 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T12:44:32,626 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T12:44:32,626 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:32,626 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T12:44:32,626 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T12:44:32,628 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T12:44:32,629 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T12:44:32,629 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:32,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:32,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T12:44:32,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T12:44:32,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:32,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:32,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T12:44:32,632 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T12:44:32,632 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:32,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:32,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T12:44:32,634 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T12:44:32,634 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:32,634 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T12:44:32,634 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T12:44:32,635 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740 2024-11-08T12:44:32,637 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740 2024-11-08T12:44:32,639 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T12:44:32,639 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T12:44:32,639 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T12:44:32,641 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T12:44:32,643 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75260684, jitterRate=0.1214715838432312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T12:44:32,643 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T12:44:32,644 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731069872626Writing region info on filesystem at 1731069872627 (+1 ms)Initializing all the Stores at 1731069872628 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069872628Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069872628Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069872628Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731069872628Cleaning up temporary data from old regions at 1731069872639 (+11 ms)Running coprocessor post-open hooks at 1731069872643 (+4 ms)Region opened successfully at 1731069872644 (+1 ms) 2024-11-08T12:44:32,646 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731069872594 2024-11-08T12:44:32,650 DEBUG [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T12:44:32,651 INFO [RS_OPEN_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T12:44:32,651 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,653 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7d22a5ea652e,41661,1731069871486, state=OPEN 2024-11-08T12:44:32,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T12:44:32,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T12:44:32,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T12:44:32,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T12:44:32,655 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:32,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:32,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:32,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:32,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T12:44:32,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T12:44:32,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7d22a5ea652e,41661,1731069871486 in 224 msec 2024-11-08T12:44:32,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T12:44:32,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 654 msec 2024-11-08T12:44:32,665 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T12:44:32,665 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T12:44:32,666 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T12:44:32,666 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d22a5ea652e,41661,1731069871486, seqNum=-1] 2024-11-08T12:44:32,667 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T12:44:32,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40329, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T12:44:32,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 775 msec 2024-11-08T12:44:32,677 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731069872677, completionTime=-1 2024-11-08T12:44:32,677 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-08T12:44:32,677 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T12:44:32,680 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-08T12:44:32,680 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731069932680 2024-11-08T12:44:32,680 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731069992680 2024-11-08T12:44:32,680 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-08T12:44:32,680 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,46881,1731069871394-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,681 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,46881,1731069871394-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,681 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,46881,1731069871394-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,681 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7d22a5ea652e:46881, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,681 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,681 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,683 DEBUG [master/7d22a5ea652e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T12:44:32,686 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.078sec 2024-11-08T12:44:32,687 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T12:44:32,687 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T12:44:32,687 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T12:44:32,687 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T12:44:32,687 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T12:44:32,687 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,46881,1731069871394-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T12:44:32,687 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,46881,1731069871394-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T12:44:32,690 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T12:44:32,690 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T12:44:32,690 INFO [master/7d22a5ea652e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7d22a5ea652e,46881,1731069871394-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T12:44:32,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c73b5a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T12:44:32,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7d22a5ea652e,46881,-1 for getting cluster id 2024-11-08T12:44:32,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T12:44:32,781 DEBUG [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1a9516c5-c300-4b5d-ba4c-437b6c2b9a44' 2024-11-08T12:44:32,781 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T12:44:32,781 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1a9516c5-c300-4b5d-ba4c-437b6c2b9a44" 2024-11-08T12:44:32,782 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af9bd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T12:44:32,782 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7d22a5ea652e,46881,-1] 2024-11-08T12:44:32,782 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T12:44:32,783 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:32,784 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T12:44:32,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea34b71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T12:44:32,786 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T12:44:32,787 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7d22a5ea652e,41661,1731069871486, seqNum=-1] 2024-11-08T12:44:32,787 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T12:44:32,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T12:44:32,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:32,792 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T12:44:32,793 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:32,793 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59bee100 2024-11-08T12:44:32,794 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T12:44:32,796 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T12:44:32,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T12:44:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC 2024-11-08T12:44:32,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T12:44:32,801 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:32,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestHBaseWalOnEC" procId is: 4 2024-11-08T12:44:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T12:44:32,804 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T12:44:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741837_1013 (size=392) 2024-11-08T12:44:32,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741837_1013 (size=392) 2024-11-08T12:44:32,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741837_1013 (size=392) 2024-11-08T12:44:32,821 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 38900ca6bcd12f018b6f339eb3eb6742, NAME => 'TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestHBaseWalOnEC', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3 2024-11-08T12:44:32,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741838_1014 (size=51) 2024-11-08T12:44:32,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741838_1014 (size=51) 2024-11-08T12:44:32,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741838_1014 (size=51) 2024-11-08T12:44:32,839 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:32,840 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1722): Closing 38900ca6bcd12f018b6f339eb3eb6742, disabling compactions & flushes 2024-11-08T12:44:32,840 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:32,840 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:32,840 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. after waiting 0 ms 2024-11-08T12:44:32,840 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:32,840 INFO [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:32,840 DEBUG [RegionOpenAndInit-TestHBaseWalOnEC-pool-0 {}] regionserver.HRegion(1676): Region close journal for 38900ca6bcd12f018b6f339eb3eb6742: Waiting for close lock at 1731069872840Disabling compacts and flushes for region at 1731069872840Disabling writes for close at 1731069872840Writing region close event to WAL at 1731069872840Closed at 1731069872840 2024-11-08T12:44:32,842 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T12:44:32,842 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742.","families":{"info":[{"qualifier":"regioninfo","vlen":50,"tag":[],"timestamp":"1731069872842"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731069872842"}]},"ts":"1731069872842"} 2024-11-08T12:44:32,846 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T12:44:32,847 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T12:44:32,848 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731069872848"}]},"ts":"1731069872848"} 2024-11-08T12:44:32,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLING in hbase:meta 2024-11-08T12:44:32,851 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {7d22a5ea652e=0} racks are {/default-rack=0} 2024-11-08T12:44:32,853 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T12:44:32,853 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T12:44:32,853 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T12:44:32,853 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T12:44:32,853 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T12:44:32,853 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T12:44:32,853 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T12:44:32,853 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T12:44:32,853 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T12:44:32,853 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T12:44:32,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=38900ca6bcd12f018b6f339eb3eb6742, ASSIGN}] 2024-11-08T12:44:32,855 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=38900ca6bcd12f018b6f339eb3eb6742, ASSIGN 2024-11-08T12:44:32,857 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=38900ca6bcd12f018b6f339eb3eb6742, ASSIGN; state=OFFLINE, location=7d22a5ea652e,45869,1731069871447; forceNewPlan=false, retain=false 2024-11-08T12:44:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T12:44:33,008 INFO [7d22a5ea652e:46881 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T12:44:33,009 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=38900ca6bcd12f018b6f339eb3eb6742, regionState=OPENING, regionLocation=7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:33,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=38900ca6bcd12f018b6f339eb3eb6742, ASSIGN because future has completed 2024-11-08T12:44:33,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38900ca6bcd12f018b6f339eb3eb6742, server=7d22a5ea652e,45869,1731069871447}] 2024-11-08T12:44:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T12:44:33,168 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T12:44:33,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39323, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T12:44:33,176 INFO [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,176 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 38900ca6bcd12f018b6f339eb3eb6742, NAME => 'TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742.', STARTKEY => '', ENDKEY => ''} 2024-11-08T12:44:33,177 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestHBaseWalOnEC 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,177 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T12:44:33,177 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,177 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,179 INFO [StoreOpener-38900ca6bcd12f018b6f339eb3eb6742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,181 INFO [StoreOpener-38900ca6bcd12f018b6f339eb3eb6742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38900ca6bcd12f018b6f339eb3eb6742 columnFamilyName cf 2024-11-08T12:44:33,181 DEBUG [StoreOpener-38900ca6bcd12f018b6f339eb3eb6742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T12:44:33,181 INFO [StoreOpener-38900ca6bcd12f018b6f339eb3eb6742-1 {}] regionserver.HStore(327): Store=38900ca6bcd12f018b6f339eb3eb6742/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T12:44:33,182 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,183 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,183 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,184 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,184 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,186 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,189 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T12:44:33,190 INFO [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 38900ca6bcd12f018b6f339eb3eb6742; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60170977, jitterRate=-0.10338257253170013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T12:44:33,190 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,191 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 38900ca6bcd12f018b6f339eb3eb6742: Running coprocessor pre-open hook at 1731069873177Writing region info on filesystem at 1731069873177Initializing all the Stores at 1731069873178 (+1 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731069873178Cleaning up temporary data from old regions at 1731069873184 (+6 ms)Running coprocessor post-open hooks at 1731069873190 (+6 ms)Region opened successfully at 1731069873191 (+1 ms) 2024-11-08T12:44:33,193 INFO [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742., pid=6, masterSystemTime=1731069873168 2024-11-08T12:44:33,196 DEBUG [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,197 INFO [RS_OPEN_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,198 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=38900ca6bcd12f018b6f339eb3eb6742, regionState=OPEN, openSeqNum=2, regionLocation=7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:33,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 38900ca6bcd12f018b6f339eb3eb6742, server=7d22a5ea652e,45869,1731069871447 because future has completed 2024-11-08T12:44:33,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T12:44:33,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 38900ca6bcd12f018b6f339eb3eb6742, server=7d22a5ea652e,45869,1731069871447 in 195 msec 2024-11-08T12:44:33,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T12:44:33,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestHBaseWalOnEC, region=38900ca6bcd12f018b6f339eb3eb6742, ASSIGN in 361 msec 2024-11-08T12:44:33,220 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T12:44:33,221 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestHBaseWalOnEC","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731069873220"}]},"ts":"1731069873220"} 2024-11-08T12:44:33,225 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestHBaseWalOnEC, state=ENABLED in hbase:meta 2024-11-08T12:44:33,226 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestHBaseWalOnEC execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T12:44:33,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestHBaseWalOnEC in 429 msec 2024-11-08T12:44:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T12:44:33,435 INFO [RPCClient-NioEventLoopGroup-6-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestHBaseWalOnEC completed 2024-11-08T12:44:33,435 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table TestHBaseWalOnEC get assigned. Timeout = 60000ms 2024-11-08T12:44:33,436 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T12:44:33,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table TestHBaseWalOnEC assigned to meta. Checking AM states. 2024-11-08T12:44:33,442 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T12:44:33,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table TestHBaseWalOnEC assigned. 2024-11-08T12:44:33,446 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestHBaseWalOnEC', row='row', locateType=CURRENT is [region=TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742., hostname=7d22a5ea652e,45869,1731069871447, seqNum=2] 2024-11-08T12:44:33,447 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T12:44:33,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T12:44:33,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestHBaseWalOnEC 2024-11-08T12:44:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC 2024-11-08T12:44:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T12:44:33,468 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_PREPARE 2024-11-08T12:44:33,470 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T12:44:33,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T12:44:33,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T12:44:33,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45869 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T12:44:33,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,626 INFO [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 38900ca6bcd12f018b6f339eb3eb6742 1/1 column families, dataSize=32 B heapSize=360 B 2024-11-08T12:44:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742/.tmp/cf/60ad812c5d4e4492ac635e3970715976 is 36, key is row/cf:cq/1731069873454/Put/seqid=0 2024-11-08T12:44:33,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741839_1015 (size=4787) 2024-11-08T12:44:33,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741839_1015 (size=4787) 2024-11-08T12:44:33,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741839_1015 (size=4787) 2024-11-08T12:44:33,656 INFO [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742/.tmp/cf/60ad812c5d4e4492ac635e3970715976 2024-11-08T12:44:33,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742/.tmp/cf/60ad812c5d4e4492ac635e3970715976 as hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742/cf/60ad812c5d4e4492ac635e3970715976 2024-11-08T12:44:33,673 INFO [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742/cf/60ad812c5d4e4492ac635e3970715976, entries=1, sequenceid=5, filesize=4.7 K 2024-11-08T12:44:33,675 INFO [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~32 B/32, heapSize ~344 B/344, currentSize=0 B/0 for 38900ca6bcd12f018b6f339eb3eb6742 in 49ms, sequenceid=5, compaction requested=false 2024-11-08T12:44:33,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 38900ca6bcd12f018b6f339eb3eb6742: 2024-11-08T12:44:33,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7d22a5ea652e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-08T12:44:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-08T12:44:33,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T12:44:33,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 207 msec 2024-11-08T12:44:33,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestHBaseWalOnEC in 217 msec 2024-11-08T12:44:33,743 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T12:44:33,744 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T12:44:33,746 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T12:44:33,746 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T12:44:33,746 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T12:44:33,746 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T12:44:33,747 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestHBaseWalOnEC 2024-11-08T12:44:33,747 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestHBaseWalOnEC Metrics about Tables on a single HBase RegionServer 2024-11-08T12:44:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46881 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T12:44:33,784 INFO [RPCClient-NioEventLoopGroup-6-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestHBaseWalOnEC completed 2024-11-08T12:44:33,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T12:44:33,789 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T12:44:33,789 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:33,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:33,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:33,790 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T12:44:33,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T12:44:33,790 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1736274792, stopped=false 2024-11-08T12:44:33,790 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7d22a5ea652e,46881,1731069871394 2024-11-08T12:44:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:33,792 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T12:44:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T12:44:33,792 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T12:44:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:33,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:33,793 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.TestHBaseWalOnEC.tearDown(TestHBaseWalOnEC.java:101) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:33,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:33,793 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d22a5ea652e,45869,1731069871447' ***** 2024-11-08T12:44:33,793 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T12:44:33,793 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d22a5ea652e,41661,1731069871486' ***** 2024-11-08T12:44:33,793 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T12:44:33,793 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:33,793 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7d22a5ea652e,46679,1731069871547' ***** 2024-11-08T12:44:33,793 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T12:44:33,794 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T12:44:33,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:33,794 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T12:44:33,794 INFO [RS:0;7d22a5ea652e:45869 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T12:44:33,794 INFO [RS:2;7d22a5ea652e:46679 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T12:44:33,794 INFO [RS:0;7d22a5ea652e:45869 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T12:44:33,794 INFO [RS:2;7d22a5ea652e:46679 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T12:44:33,794 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T12:44:33,794 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(959): stopping server 7d22a5ea652e,46679,1731069871547 2024-11-08T12:44:33,794 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(3091): Received CLOSE for 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,794 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T12:44:33,794 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T12:44:33,794 INFO [RS:2;7d22a5ea652e:46679 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;7d22a5ea652e:46679. 2024-11-08T12:44:33,794 DEBUG [RS:2;7d22a5ea652e:46679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:33,794 DEBUG [RS:2;7d22a5ea652e:46679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:33,794 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(976): stopping server 7d22a5ea652e,46679,1731069871547; all regions closed. 2024-11-08T12:44:33,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:33,795 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(959): stopping server 7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7d22a5ea652e:41661. 2024-11-08T12:44:33,795 DEBUG [RS:1;7d22a5ea652e:41661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:33,795 DEBUG [RS:1;7d22a5ea652e:41661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:33,795 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T12:44:33,795 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T12:44:33,795 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T12:44:33,796 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T12:44:33,796 DEBUG [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T12:44:33,796 DEBUG [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T12:44:33,796 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T12:44:33,796 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T12:44:33,796 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T12:44:33,796 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T12:44:33,796 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T12:44:33,797 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB 2024-11-08T12:44:33,797 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(959): stopping server 7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:33,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,797 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T12:44:33,797 INFO [RS:0;7d22a5ea652e:45869 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7d22a5ea652e:45869. 2024-11-08T12:44:33,797 DEBUG [RS:0;7d22a5ea652e:45869 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T12:44:33,797 DEBUG [RS:0;7d22a5ea652e:45869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:33,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,797 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T12:44:33,797 DEBUG [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1325): Online Regions={38900ca6bcd12f018b6f339eb3eb6742=TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742.} 2024-11-08T12:44:33,797 DEBUG [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1351): Waiting on 38900ca6bcd12f018b6f339eb3eb6742 2024-11-08T12:44:33,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,798 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 38900ca6bcd12f018b6f339eb3eb6742, disabling compactions & flushes 2024-11-08T12:44:33,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,798 INFO [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,798 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,798 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. after waiting 0 ms 2024-11-08T12:44:33,798 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741835_1011 (size=93) 2024-11-08T12:44:33,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741835_1011 (size=93) 2024-11-08T12:44:33,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741835_1011 (size=93) 2024-11-08T12:44:33,808 DEBUG [RS:2;7d22a5ea652e:46679 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/oldWALs 2024-11-08T12:44:33,808 INFO [RS:2;7d22a5ea652e:46679 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7d22a5ea652e%2C46679%2C1731069871547:(num 1731069872219) 2024-11-08T12:44:33,808 DEBUG [RS:2;7d22a5ea652e:46679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:33,808 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:33,808 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T12:44:33,808 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.ChoreService(370): Chore service for: regionserver/7d22a5ea652e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T12:44:33,808 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T12:44:33,808 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T12:44:33,808 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T12:44:33,809 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T12:44:33,809 INFO [RS:2;7d22a5ea652e:46679 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46679 2024-11-08T12:44:33,809 INFO [regionserver/7d22a5ea652e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T12:44:33,810 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/default/TestHBaseWalOnEC/38900ca6bcd12f018b6f339eb3eb6742/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T12:44:33,811 INFO [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d22a5ea652e,46679,1731069871547 2024-11-08T12:44:33,811 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 38900ca6bcd12f018b6f339eb3eb6742: Waiting for close lock at 1731069873798Running coprocessor pre-close hooks at 1731069873798Disabling compacts and flushes for region at 1731069873798Disabling writes for close at 1731069873798Writing region close event to WAL at 1731069873800 (+2 ms)Running coprocessor post-close hooks at 1731069873811 (+11 ms)Closed at 1731069873811 2024-11-08T12:44:33,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T12:44:33,811 DEBUG [RS_CLOSE_REGION-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742. 2024-11-08T12:44:33,812 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T12:44:33,812 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d22a5ea652e,46679,1731069871547] 2024-11-08T12:44:33,814 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d22a5ea652e,46679,1731069871547 already deleted, retry=false 2024-11-08T12:44:33,814 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d22a5ea652e,46679,1731069871547 expired; onlineServers=2 2024-11-08T12:44:33,826 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/info/e4745971cea24f1186a0cd2c8e7faf07 is 153, key is TestHBaseWalOnEC,,1731069872796.38900ca6bcd12f018b6f339eb3eb6742./info:regioninfo/1731069873198/Put/seqid=0 2024-11-08T12:44:33,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741840_1016 (size=6637) 2024-11-08T12:44:33,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741840_1016 (size=6637) 2024-11-08T12:44:33,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741840_1016 (size=6637) 2024-11-08T12:44:33,836 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/info/e4745971cea24f1186a0cd2c8e7faf07 2024-11-08T12:44:33,860 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/ns/9fe8a073925b4839840e2e3402b573fe is 43, key is default/ns:d/1731069872669/Put/seqid=0 2024-11-08T12:44:33,866 INFO [regionserver/7d22a5ea652e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741841_1017 (size=5153) 2024-11-08T12:44:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741841_1017 (size=5153) 2024-11-08T12:44:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741841_1017 (size=5153) 2024-11-08T12:44:33,869 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/ns/9fe8a073925b4839840e2e3402b573fe 2024-11-08T12:44:33,873 INFO [regionserver/7d22a5ea652e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:33,873 INFO [regionserver/7d22a5ea652e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:33,899 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/table/e70a1da44df24984aaf8f28887edb978 is 52, key is TestHBaseWalOnEC/table:state/1731069873220/Put/seqid=0 2024-11-08T12:44:33,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741842_1018 (size=5249) 2024-11-08T12:44:33,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741842_1018 (size=5249) 2024-11-08T12:44:33,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741842_1018 (size=5249) 2024-11-08T12:44:33,907 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=96 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/table/e70a1da44df24984aaf8f28887edb978 2024-11-08T12:44:33,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:33,914 INFO [RS:2;7d22a5ea652e:46679 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T12:44:33,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46679-0x1001f6c0faa0003, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:33,914 INFO [RS:2;7d22a5ea652e:46679 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d22a5ea652e,46679,1731069871547; zookeeper connection closed. 2024-11-08T12:44:33,914 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@31a47b60 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@31a47b60 2024-11-08T12:44:33,915 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/info/e4745971cea24f1186a0cd2c8e7faf07 as hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/info/e4745971cea24f1186a0cd2c8e7faf07 2024-11-08T12:44:33,934 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/info/e4745971cea24f1186a0cd2c8e7faf07, entries=10, sequenceid=11, filesize=6.5 K 2024-11-08T12:44:33,937 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/ns/9fe8a073925b4839840e2e3402b573fe as hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/ns/9fe8a073925b4839840e2e3402b573fe 2024-11-08T12:44:33,946 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/ns/9fe8a073925b4839840e2e3402b573fe, entries=2, sequenceid=11, filesize=5.0 K 2024-11-08T12:44:33,947 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/.tmp/table/e70a1da44df24984aaf8f28887edb978 as hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/table/e70a1da44df24984aaf8f28887edb978 2024-11-08T12:44:33,956 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/table/e70a1da44df24984aaf8f28887edb978, entries=2, sequenceid=11, filesize=5.1 K 2024-11-08T12:44:33,958 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 161ms, sequenceid=11, compaction requested=false 2024-11-08T12:44:33,964 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-08T12:44:33,966 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T12:44:33,966 INFO [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T12:44:33,966 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731069873796Running coprocessor pre-close hooks at 1731069873796Disabling compacts and flushes for region at 1731069873796Disabling writes for close at 1731069873796Obtaining lock to block concurrent updates at 1731069873797 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731069873797Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1377, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1731069873798 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731069873800 (+2 ms)Flushing 1588230740/info: creating writer at 1731069873800Flushing 1588230740/info: appending metadata at 1731069873826 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731069873826Flushing 1588230740/ns: creating writer at 1731069873843 (+17 ms)Flushing 1588230740/ns: appending metadata at 1731069873859 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731069873859Flushing 1588230740/table: creating writer at 1731069873876 (+17 ms)Flushing 1588230740/table: appending metadata at 1731069873898 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731069873898Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@478553cd: reopening flushed file at 1731069873914 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24673a2d: reopening flushed file at 1731069873935 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4580419d: reopening flushed file at 1731069873946 (+11 ms)Finished flush of dataSize ~1.34 KB/1377, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 161ms, sequenceid=11, compaction requested=false at 1731069873958 (+12 ms)Writing region close event to WAL at 1731069873959 (+1 ms)Running coprocessor post-close hooks at 1731069873965 (+6 ms)Closed at 1731069873966 (+1 ms) 2024-11-08T12:44:33,966 DEBUG [RS_CLOSE_META-regionserver/7d22a5ea652e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T12:44:33,996 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(976): stopping server 7d22a5ea652e,41661,1731069871486; all regions closed. 2024-11-08T12:44:33,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,998 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(976): stopping server 7d22a5ea652e,45869,1731069871447; all regions closed. 2024-11-08T12:44:33,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:33,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741836_1012 (size=2751) 2024-11-08T12:44:34,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741836_1012 (size=2751) 2024-11-08T12:44:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741834_1010 (size=1298) 2024-11-08T12:44:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741836_1012 (size=2751) 2024-11-08T12:44:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741834_1010 (size=1298) 2024-11-08T12:44:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741834_1010 (size=1298) 2024-11-08T12:44:34,005 DEBUG [RS:0;7d22a5ea652e:45869 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/oldWALs 2024-11-08T12:44:34,005 INFO [RS:0;7d22a5ea652e:45869 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7d22a5ea652e%2C45869%2C1731069871447:(num 1731069872225) 2024-11-08T12:44:34,005 DEBUG [RS:0;7d22a5ea652e:45869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:34,005 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:34,005 DEBUG [RS:1;7d22a5ea652e:41661 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/oldWALs 2024-11-08T12:44:34,005 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T12:44:34,005 INFO [RS:1;7d22a5ea652e:41661 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7d22a5ea652e%2C41661%2C1731069871486.meta:.meta(num 1731069872606) 2024-11-08T12:44:34,005 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.ChoreService(370): Chore service for: regionserver/7d22a5ea652e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T12:44:34,006 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T12:44:34,006 INFO [regionserver/7d22a5ea652e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T12:44:34,006 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T12:44:34,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,006 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T12:44:34,006 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T12:44:34,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,006 INFO [RS:0;7d22a5ea652e:45869 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45869 2024-11-08T12:44:34,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T12:44:34,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d22a5ea652e,45869,1731069871447 2024-11-08T12:44:34,008 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T12:44:34,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741833_1009 (size=93) 2024-11-08T12:44:34,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741833_1009 (size=93) 2024-11-08T12:44:34,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741833_1009 (size=93) 2024-11-08T12:44:34,010 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d22a5ea652e,45869,1731069871447] 2024-11-08T12:44:34,012 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d22a5ea652e,45869,1731069871447 already deleted, retry=false 2024-11-08T12:44:34,012 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d22a5ea652e,45869,1731069871447 expired; onlineServers=1 2024-11-08T12:44:34,012 DEBUG [RS:1;7d22a5ea652e:41661 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/oldWALs 2024-11-08T12:44:34,012 INFO [RS:1;7d22a5ea652e:41661 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7d22a5ea652e%2C41661%2C1731069871486:(num 1731069872223) 2024-11-08T12:44:34,012 DEBUG [RS:1;7d22a5ea652e:41661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T12:44:34,012 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T12:44:34,012 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T12:44:34,012 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.ChoreService(370): Chore service for: regionserver/7d22a5ea652e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T12:44:34,012 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T12:44:34,012 INFO [regionserver/7d22a5ea652e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T12:44:34,013 INFO [RS:1;7d22a5ea652e:41661 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41661 2024-11-08T12:44:34,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7d22a5ea652e,41661,1731069871486 2024-11-08T12:44:34,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T12:44:34,014 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T12:44:34,016 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7d22a5ea652e,41661,1731069871486] 2024-11-08T12:44:34,017 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7d22a5ea652e,41661,1731069871486 already deleted, retry=false 2024-11-08T12:44:34,017 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7d22a5ea652e,41661,1731069871486 expired; onlineServers=0 2024-11-08T12:44:34,017 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7d22a5ea652e,46881,1731069871394' ***** 2024-11-08T12:44:34,017 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T12:44:34,018 INFO [M:0;7d22a5ea652e:46881 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T12:44:34,018 INFO [M:0;7d22a5ea652e:46881 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T12:44:34,018 DEBUG [M:0;7d22a5ea652e:46881 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T12:44:34,018 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T12:44:34,018 DEBUG [M:0;7d22a5ea652e:46881 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T12:44:34,018 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.large.0-1731069871908 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.large.0-1731069871908,5,FailOnTimeoutGroup] 2024-11-08T12:44:34,018 DEBUG [master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.small.0-1731069871908 {}] cleaner.HFileCleaner(306): Exit Thread[master/7d22a5ea652e:0:becomeActiveMaster-HFileCleaner.small.0-1731069871908,5,FailOnTimeoutGroup] 2024-11-08T12:44:34,018 INFO [M:0;7d22a5ea652e:46881 {}] hbase.ChoreService(370): Chore service for: master/7d22a5ea652e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T12:44:34,018 INFO [M:0;7d22a5ea652e:46881 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T12:44:34,018 DEBUG [M:0;7d22a5ea652e:46881 {}] master.HMaster(1795): Stopping service threads 2024-11-08T12:44:34,018 INFO [M:0;7d22a5ea652e:46881 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T12:44:34,018 INFO [M:0;7d22a5ea652e:46881 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T12:44:34,019 INFO [M:0;7d22a5ea652e:46881 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T12:44:34,019 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T12:44:34,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T12:44:34,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T12:44:34,020 DEBUG [M:0;7d22a5ea652e:46881 {}] zookeeper.ZKUtil(347): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T12:44:34,020 WARN [M:0;7d22a5ea652e:46881 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T12:44:34,020 INFO [M:0;7d22a5ea652e:46881 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/.lastflushedseqids 2024-11-08T12:44:34,022 WARN [IPC Server handler 4 on default port 39575 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:34,022 WARN [IPC Server handler 4 on default port 39575 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:34,022 WARN [IPC Server handler 4 on default port 39575 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741843_1019 (size=127) 2024-11-08T12:44:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741843_1019 (size=127) 2024-11-08T12:44:34,029 INFO [M:0;7d22a5ea652e:46881 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T12:44:34,030 INFO [M:0;7d22a5ea652e:46881 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T12:44:34,030 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T12:44:34,030 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:34,030 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:34,030 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T12:44:34,030 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:34,030 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=26.83 KB heapSize=34.12 KB 2024-11-08T12:44:34,049 DEBUG [M:0;7d22a5ea652e:46881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a25df60186a44b378bfdb4033b5e4d4d is 82, key is hbase:meta,,1/info:regioninfo/1731069872651/Put/seqid=0 2024-11-08T12:44:34,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741844_1020 (size=5672) 2024-11-08T12:44:34,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741844_1020 (size=5672) 2024-11-08T12:44:34,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741844_1020 (size=5672) 2024-11-08T12:44:34,058 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a25df60186a44b378bfdb4033b5e4d4d 2024-11-08T12:44:34,082 DEBUG [M:0;7d22a5ea652e:46881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/53a944e2bb0b4aedbb562f611a8a30b5 is 748, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731069873228/Put/seqid=0 2024-11-08T12:44:34,083 WARN [IPC Server handler 0 on default port 39575 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:34,083 WARN [IPC Server handler 0 on default port 39575 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:34,083 WARN [IPC Server handler 0 on default port 39575 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:34,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741845_1021 (size=6439) 2024-11-08T12:44:34,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741845_1021 (size=6439) 2024-11-08T12:44:34,089 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.14 KB at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/53a944e2bb0b4aedbb562f611a8a30b5 2024-11-08T12:44:34,110 INFO [RS:0;7d22a5ea652e:45869 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T12:44:34,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:34,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45869-0x1001f6c0faa0001, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:34,111 INFO [RS:0;7d22a5ea652e:45869 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d22a5ea652e,45869,1731069871447; zookeeper connection closed. 2024-11-08T12:44:34,111 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b511a08 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b511a08 2024-11-08T12:44:34,113 DEBUG [M:0;7d22a5ea652e:46881 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab117cdbf38a4852983bfcd31726f3eb is 69, key is 7d22a5ea652e,41661,1731069871486/rs:state/1731069872023/Put/seqid=0 2024-11-08T12:44:34,115 WARN [IPC Server handler 1 on default port 39575 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-08T12:44:34,115 WARN [IPC Server handler 1 on default port 39575 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-08T12:44:34,115 WARN [IPC Server handler 1 on default port 39575 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-08T12:44:34,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:34,116 INFO [RS:1;7d22a5ea652e:41661 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T12:44:34,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41661-0x1001f6c0faa0002, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:34,116 INFO [RS:1;7d22a5ea652e:41661 {}] regionserver.HRegionServer(1031): Exiting; stopping=7d22a5ea652e,41661,1731069871486; zookeeper connection closed. 2024-11-08T12:44:34,117 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27b25242 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27b25242 2024-11-08T12:44:34,117 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T12:44:34,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741846_1022 (size=5294) 2024-11-08T12:44:34,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741846_1022 (size=5294) 2024-11-08T12:44:34,122 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=195 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab117cdbf38a4852983bfcd31726f3eb 2024-11-08T12:44:34,131 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a25df60186a44b378bfdb4033b5e4d4d as hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a25df60186a44b378bfdb4033b5e4d4d 2024-11-08T12:44:34,141 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a25df60186a44b378bfdb4033b5e4d4d, entries=8, sequenceid=72, filesize=5.5 K 2024-11-08T12:44:34,143 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/53a944e2bb0b4aedbb562f611a8a30b5 as hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/53a944e2bb0b4aedbb562f611a8a30b5 2024-11-08T12:44:34,152 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/53a944e2bb0b4aedbb562f611a8a30b5, entries=8, sequenceid=72, filesize=6.3 K 2024-11-08T12:44:34,153 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ab117cdbf38a4852983bfcd31726f3eb as hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ab117cdbf38a4852983bfcd31726f3eb 2024-11-08T12:44:34,160 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39575/user/jenkins/test-data/ae104e6e-0374-767b-f987-1536429164e3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ab117cdbf38a4852983bfcd31726f3eb, entries=3, sequenceid=72, filesize=5.2 K 2024-11-08T12:44:34,162 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.83 KB/27471, heapSize ~33.82 KB/34632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 132ms, sequenceid=72, compaction requested=false 2024-11-08T12:44:34,163 INFO [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T12:44:34,164 DEBUG [M:0;7d22a5ea652e:46881 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731069874030Disabling compacts and flushes for region at 1731069874030Disabling writes for close at 1731069874030Obtaining lock to block concurrent updates at 1731069874030Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731069874030Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=27471, getHeapSize=34872, getOffHeapSize=0, getCellsCount=85 at 1731069874030Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731069874031 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731069874031Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731069874049 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731069874049Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731069874065 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731069874082 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731069874082Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731069874095 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731069874113 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731069874113Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2725f1c3: reopening flushed file at 1731069874130 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ce41b4: reopening flushed file at 1731069874142 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dcc6c20: reopening flushed file at 1731069874152 (+10 ms)Finished flush of dataSize ~26.83 KB/27471, heapSize ~33.82 KB/34632, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 132ms, sequenceid=72, compaction requested=false at 1731069874162 (+10 ms)Writing region close event to WAL at 1731069874163 (+1 ms)Closed at 1731069874163 2024-11-08T12:44:34,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-08T12:44:34,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35653 is added to blk_1073741830_1006 (size=32674) 2024-11-08T12:44:34,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42227 is added to blk_1073741830_1006 (size=32674) 2024-11-08T12:44:34,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741830_1006 (size=32674) 2024-11-08T12:44:34,169 INFO [M:0;7d22a5ea652e:46881 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T12:44:34,169 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T12:44:34,169 INFO [M:0;7d22a5ea652e:46881 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46881 2024-11-08T12:44:34,169 INFO [M:0;7d22a5ea652e:46881 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T12:44:34,272 INFO [M:0;7d22a5ea652e:46881 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T12:44:34,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:34,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46881-0x1001f6c0faa0000, quorum=127.0.0.1:51233, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T12:44:34,274 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T12:44:34,275 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42279afb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:34,275 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776d8022{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T12:44:34,275 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T12:44:34,275 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41154aac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T12:44:34,275 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@638f7c01{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir/,STOPPED} 2024-11-08T12:44:34,277 WARN [BP-549379526-172.17.0.3-1731069870368 heartbeating to localhost/127.0.0.1:39575 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T12:44:34,277 WARN [BP-549379526-172.17.0.3-1731069870368 heartbeating to localhost/127.0.0.1:39575 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-549379526-172.17.0.3-1731069870368 (Datanode Uuid 44007310-83ab-4f61-a09c-a75c8f0e1a3e) service to localhost/127.0.0.1:39575 2024-11-08T12:44:34,278 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data5/current/BP-549379526-172.17.0.3-1731069870368 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:34,278 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data6/current/BP-549379526-172.17.0.3-1731069870368 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:34,278 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T12:44:34,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26cf3f54{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:34,283 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10498144{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T12:44:34,283 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T12:44:34,283 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b034d27{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T12:44:34,283 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3dc09a89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir/,STOPPED} 2024-11-08T12:44:34,284 WARN [BP-549379526-172.17.0.3-1731069870368 heartbeating to localhost/127.0.0.1:39575 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T12:44:34,284 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T12:44:34,284 WARN [BP-549379526-172.17.0.3-1731069870368 heartbeating to localhost/127.0.0.1:39575 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-549379526-172.17.0.3-1731069870368 (Datanode Uuid 70cc5c95-972b-4c66-8c75-887e1111027e) service to localhost/127.0.0.1:39575 2024-11-08T12:44:34,284 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T12:44:34,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data3/current/BP-549379526-172.17.0.3-1731069870368 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:34,285 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data4/current/BP-549379526-172.17.0.3-1731069870368 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:34,285 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T12:44:34,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7005d2e9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T12:44:34,288 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6496cd5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T12:44:34,288 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T12:44:34,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54931008{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T12:44:34,288 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33ecba79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir/,STOPPED} 2024-11-08T12:44:34,289 WARN [BP-549379526-172.17.0.3-1731069870368 heartbeating to localhost/127.0.0.1:39575 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T12:44:34,289 WARN [BP-549379526-172.17.0.3-1731069870368 heartbeating to localhost/127.0.0.1:39575 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-549379526-172.17.0.3-1731069870368 (Datanode Uuid bf3ca83b-feac-45ef-ba4e-b96c843104db) service to localhost/127.0.0.1:39575 2024-11-08T12:44:34,289 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T12:44:34,289 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T12:44:34,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data1/current/BP-549379526-172.17.0.3-1731069870368 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:34,290 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/cluster_e465defd-3c65-97b1-2641-8311b9c1efb2/data/data2/current/BP-549379526-172.17.0.3-1731069870368 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T12:44:34,290 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T12:44:34,297 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4b64f5ed{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T12:44:34,298 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c00a7dd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T12:44:34,298 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T12:44:34,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68969217{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T12:44:34,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18bcac98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b47ea669-354e-1d34-0b93-ff77a33d8b5c/hadoop.log.dir/,STOPPED} 2024-11-08T12:44:34,307 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T12:44:34,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T12:44:34,349 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestHBaseWalOnEC#testReadWrite[1] Thread=150 (was 88) - Thread LEAK? -, OpenFileDescriptor=518 (was 435) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=465 (was 454) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1387 (was 1688)