2024-11-25 08:19:27,234 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c137fd5 2024-11-25 08:19:27,252 main DEBUG Took 0.014838 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 08:19:27,252 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 08:19:27,253 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 08:19:27,254 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 08:19:27,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,267 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 08:19:27,283 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,286 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,287 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,289 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,290 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,291 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,292 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,293 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,295 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,296 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,297 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,298 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 08:19:27,299 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,299 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 08:19:27,301 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 08:19:27,303 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 08:19:27,305 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 08:19:27,305 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 08:19:27,307 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 08:19:27,307 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 08:19:27,317 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 08:19:27,320 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 08:19:27,322 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 08:19:27,322 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 08:19:27,323 main DEBUG createAppenders(={Console}) 2024-11-25 08:19:27,323 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c137fd5 initialized 2024-11-25 08:19:27,324 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c137fd5 2024-11-25 08:19:27,324 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c137fd5 OK. 2024-11-25 08:19:27,325 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 08:19:27,325 main DEBUG OutputStream closed 2024-11-25 08:19:27,325 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 08:19:27,325 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 08:19:27,326 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@14555e0a OK 2024-11-25 08:19:27,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 08:19:27,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 08:19:27,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 08:19:27,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 08:19:27,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 08:19:27,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 08:19:27,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 08:19:27,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 08:19:27,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 08:19:27,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 08:19:27,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 08:19:27,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 08:19:27,411 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 08:19:27,411 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 08:19:27,411 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 08:19:27,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 08:19:27,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 08:19:27,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 08:19:27,415 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 08:19:27,415 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5f9be66c) with optional ClassLoader: null 2024-11-25 08:19:27,416 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 08:19:27,416 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5f9be66c] started OK. 2024-11-25T08:19:27,463 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-25 08:19:27,468 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 08:19:27,468 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T08:19:27,784 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828 2024-11-25T08:19:27,788 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-25T08:19:27,790 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:19:27,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T08:19:27,809 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def, deleteOnExit=true 2024-11-25T08:19:27,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T08:19:27,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/test.cache.data in system properties and HBase conf 2024-11-25T08:19:27,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T08:19:27,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir in system properties and HBase conf 2024-11-25T08:19:27,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T08:19:27,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T08:19:27,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T08:19:27,978 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T08:19:28,108 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T08:19:28,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T08:19:28,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T08:19:28,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T08:19:28,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T08:19:28,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T08:19:28,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T08:19:28,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T08:19:28,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T08:19:28,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T08:19:28,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/nfs.dump.dir in system properties and HBase conf 2024-11-25T08:19:28,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/java.io.tmpdir in system properties and HBase conf 2024-11-25T08:19:28,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T08:19:28,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T08:19:28,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T08:19:30,769 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T08:19:30,861 INFO [Time-limited test {}] log.Log(170): Logging initialized @4998ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T08:19:30,973 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:31,076 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T08:19:31,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T08:19:31,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T08:19:31,109 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T08:19:31,132 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:31,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f83a6b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,AVAILABLE} 2024-11-25T08:19:31,145 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e61513{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T08:19:31,391 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ee33b9d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/java.io.tmpdir/jetty-localhost-34567-hadoop-hdfs-3_4_1-tests_jar-_-any-12615783463012129549/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T08:19:31,398 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a655ef8{HTTP/1.1, (http/1.1)}{localhost:34567} 2024-11-25T08:19:31,398 INFO [Time-limited test {}] server.Server(415): Started @5537ms 2024-11-25T08:19:32,308 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:32,315 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T08:19:32,316 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T08:19:32,316 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T08:19:32,317 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T08:19:32,318 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41edcc56{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,AVAILABLE} 2024-11-25T08:19:32,318 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c1aa6bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T08:19:32,441 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6acf7532{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/java.io.tmpdir/jetty-localhost-39479-hadoop-hdfs-3_4_1-tests_jar-_-any-13124138469879060615/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T08:19:32,442 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74250dde{HTTP/1.1, (http/1.1)}{localhost:39479} 2024-11-25T08:19:32,443 INFO [Time-limited test {}] server.Server(415): Started @6581ms 2024-11-25T08:19:32,508 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T08:19:33,968 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/data/data1/current/BP-814101347-172.17.0.2-1732522768861/current, will proceed with Du for space computation calculation, 2024-11-25T08:19:33,968 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/data/data2/current/BP-814101347-172.17.0.2-1732522768861/current, will proceed with Du for space computation calculation, 2024-11-25T08:19:33,999 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T08:19:34,056 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x636219aa32fa77d1 with lease ID 0x8aff7cf881071d01: Processing first storage report for DS-e8d3642a-6513-4e62-84b7-0aaa2923d333 from datanode DatanodeRegistration(127.0.0.1:43131, datanodeUuid=ccdb6c77-1653-4576-9bc7-57cbc5a32064, infoPort=36805, infoSecurePort=0, ipcPort=39561, storageInfo=lv=-57;cid=testClusterID;nsid=1988896260;c=1732522768861) 2024-11-25T08:19:34,057 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x636219aa32fa77d1 with lease ID 0x8aff7cf881071d01: from storage DS-e8d3642a-6513-4e62-84b7-0aaa2923d333 node DatanodeRegistration(127.0.0.1:43131, datanodeUuid=ccdb6c77-1653-4576-9bc7-57cbc5a32064, infoPort=36805, infoSecurePort=0, ipcPort=39561, storageInfo=lv=-57;cid=testClusterID;nsid=1988896260;c=1732522768861), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-25T08:19:34,058 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x636219aa32fa77d1 with lease ID 0x8aff7cf881071d01: Processing first storage report for DS-31f1bee7-a312-49cc-949d-9bf59e6dcb01 from datanode DatanodeRegistration(127.0.0.1:43131, datanodeUuid=ccdb6c77-1653-4576-9bc7-57cbc5a32064, infoPort=36805, infoSecurePort=0, ipcPort=39561, storageInfo=lv=-57;cid=testClusterID;nsid=1988896260;c=1732522768861) 2024-11-25T08:19:34,058 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x636219aa32fa77d1 with lease ID 0x8aff7cf881071d01: from storage DS-31f1bee7-a312-49cc-949d-9bf59e6dcb01 node DatanodeRegistration(127.0.0.1:43131, datanodeUuid=ccdb6c77-1653-4576-9bc7-57cbc5a32064, infoPort=36805, infoSecurePort=0, ipcPort=39561, storageInfo=lv=-57;cid=testClusterID;nsid=1988896260;c=1732522768861), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T08:19:34,095 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828 2024-11-25T08:19:34,208 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/zookeeper_0, clientPort=63891, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T08:19:34,220 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63891 2024-11-25T08:19:34,232 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T08:19:34,235 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T08:19:34,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741825_1001 (size=7) 2024-11-25T08:19:34,870 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 with version=8 2024-11-25T08:19:34,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/hbase-staging 2024-11-25T08:19:34,883 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-25T08:19:34,883 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-25T08:19:34,883 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-25T08:19:34,883 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-25T08:19:34,966 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T08:19:35,267 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5eb3d201e8c9:0 server-side Connection retries=45 2024-11-25T08:19:35,278 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T08:19:35,279 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T08:19:35,283 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T08:19:35,283 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T08:19:35,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T08:19:35,426 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T08:19:35,493 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T08:19:35,502 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T08:19:35,506 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T08:19:35,533 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5549 (auto-detected) 2024-11-25T08:19:35,534 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T08:19:35,554 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44851 2024-11-25T08:19:35,578 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44851 connecting to ZooKeeper ensemble=127.0.0.1:63891 2024-11-25T08:19:35,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:448510x0, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T08:19:35,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44851-0x1014e35793c0000 connected 2024-11-25T08:19:36,053 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T08:19:36,056 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T08:19:36,066 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-25T08:19:36,089 INFO [Time-limited test {}] log.Log(170): Logging initialized @10227ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T08:19:36,136 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-25T08:19:36,136 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-25T08:19:36,137 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-25T08:19:36,137 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-25T08:19:36,138 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-25T08:19:36,139 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-25T08:19:36,140 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-25T08:19:36,140 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-25T08:19:36,142 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-25T08:19:36,200 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 43797 2024-11-25T08:19:36,201 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-25T08:19:36,214 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-25T08:19:36,215 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@38ae1acb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,AVAILABLE} 2024-11-25T08:19:36,216 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-25T08:19:36,216 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6936b07c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-25T08:19:36,261 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-25T08:19:36,269 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T08:19:36,269 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T08:19:36,270 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T08:19:36,273 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-25T08:19:37,060 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@71a9493b{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-25T08:19:37,064 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7e3adfd7{HTTP/1.1, (http/1.1)}{0.0.0.0:43797} 2024-11-25T08:19:37,064 INFO [Time-limited test {}] server.Server(415): Started @11203ms 2024-11-25T08:19:37,068 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927, hbase.cluster.distributed=false 2024-11-25T08:19:37,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-25T08:19:37,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44851 2024-11-25T08:19:37,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44851 2024-11-25T08:19:37,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44851 2024-11-25T08:19:37,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44851 2024-11-25T08:19:37,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44851 2024-11-25T08:19:37,222 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5eb3d201e8c9:0 server-side Connection retries=45 2024-11-25T08:19:37,225 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T08:19:37,225 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T08:19:37,225 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T08:19:37,226 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T08:19:37,226 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T08:19:37,230 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T08:19:37,232 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T08:19:37,233 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43089 2024-11-25T08:19:37,235 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43089 connecting to ZooKeeper ensemble=127.0.0.1:63891 2024-11-25T08:19:37,236 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T08:19:37,239 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T08:19:37,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430890x0, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T08:19:37,283 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:430890x0, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-25T08:19:37,283 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43089-0x1014e35793c0001 connected 2024-11-25T08:19:37,285 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-25T08:19:37,285 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-25T08:19:37,286 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-25T08:19:37,286 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-25T08:19:37,286 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-25T08:19:37,287 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-25T08:19:37,287 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-25T08:19:37,288 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-25T08:19:37,288 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-25T08:19:37,290 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 43209 2024-11-25T08:19:37,290 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-25T08:19:37,292 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-25T08:19:37,292 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@e73d64a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,AVAILABLE} 2024-11-25T08:19:37,293 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-25T08:19:37,293 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@19e753c2{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-25T08:19:37,312 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-25T08:19:37,313 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T08:19:37,313 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T08:19:37,313 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T08:19:37,314 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-25T08:19:37,319 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@2f5ab22c{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-25T08:19:37,320 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@467e5a93{HTTP/1.1, (http/1.1)}{0.0.0.0:43209} 2024-11-25T08:19:37,320 INFO [Time-limited test {}] server.Server(415): Started @11459ms 2024-11-25T08:19:37,326 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T08:19:37,336 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T08:19:37,339 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-25T08:19:37,346 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-25T08:19:37,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43089 2024-11-25T08:19:37,347 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43089 2024-11-25T08:19:37,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43089 2024-11-25T08:19:37,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43089 2024-11-25T08:19:37,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43089 2024-11-25T08:19:37,376 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5eb3d201e8c9:44851 2024-11-25T08:19:37,379 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-25T08:19:37,385 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@3007dbc3{HTTP/1.1, (http/1.1)}{0.0.0.0:44321} 2024-11-25T08:19:37,385 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] server.Server(415): Started @11524ms 2024-11-25T08:19:37,385 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:37,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-25T08:19:37,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-25T08:19:37,423 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:37,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:37,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-25T08:19:37,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:37,490 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-25T08:19:37,492 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/5eb3d201e8c9,44851,1732522775045 from backup master directory 2024-11-25T08:19:37,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:37,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-25T08:19:37,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-25T08:19:37,536 WARN [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T08:19:37,536 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:37,538 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T08:19:37,540 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T08:19:37,595 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/hbase.id] with ID: 20ef2d0a-a99b-4ad7-a633-df33eb0156be 2024-11-25T08:19:37,596 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.tmp/hbase.id 2024-11-25T08:19:37,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741826_1002 (size=42) 2024-11-25T08:19:38,008 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.tmp/hbase.id]:[hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/hbase.id] 2024-11-25T08:19:38,055 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T08:19:38,060 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T08:19:38,081 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-25T08:19:38,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:38,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:38,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741827_1003 (size=196) 2024-11-25T08:19:38,544 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:19:38,547 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T08:19:38,554 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-25T08:19:38,557 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-25T08:19:38,569 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T08:19:38,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741828_1004 (size=1189) 2024-11-25T08:19:39,025 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store 2024-11-25T08:19:39,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741829_1005 (size=34) 2024-11-25T08:19:39,452 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T08:19:39,455 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:39,457 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T08:19:39,457 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T08:19:39,457 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T08:19:39,459 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T08:19:39,459 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T08:19:39,459 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T08:19:39,460 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732522779457Disabling compacts and flushes for region at 1732522779457Disabling writes for close at 1732522779459 (+2 ms)Writing region close event to WAL at 1732522779459Closed at 1732522779459 2024-11-25T08:19:39,462 WARN [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/.initializing 2024-11-25T08:19:39,463 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/WALs/5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:39,468 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T08:19:39,474 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 5eb3d201e8c9,44851,1732522775045.regiongroup-0 2024-11-25T08:19:39,493 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C44851%2C1732522775045.5eb3d201e8c9%2C44851%2C1732522775045.regiongroup-0, suffix=, logDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/WALs/5eb3d201e8c9,44851,1732522775045, archiveDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/oldWALs, maxLogs=10 2024-11-25T08:19:39,524 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/WALs/5eb3d201e8c9,44851,1732522775045/5eb3d201e8c9%2C44851%2C1732522775045.5eb3d201e8c9%2C44851%2C1732522775045.regiongroup-0.1732522779499, exclude list is [], retry=0 2024-11-25T08:19:39,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:19:39,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T08:19:39,583 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/WALs/5eb3d201e8c9,44851,1732522775045/5eb3d201e8c9%2C44851%2C1732522775045.5eb3d201e8c9%2C44851%2C1732522775045.regiongroup-0.1732522779499 2024-11-25T08:19:39,584 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:19:39,584 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:39,585 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:39,588 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,589 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T08:19:39,665 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:39,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:39,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T08:19:39,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:39,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:39,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T08:19:39,677 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:39,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:39,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T08:19:39,683 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:39,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:39,684 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,689 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,690 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,698 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,699 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,703 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T08:19:39,707 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T08:19:39,712 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T08:19:39,714 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11155806880, jitterRate=0.03896547853946686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T08:19:39,724 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732522779606Initializing all the Stores at 1732522779608 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732522779609 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522779610 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522779611 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522779611Cleaning up temporary data from old regions at 1732522779699 (+88 ms)Region opened successfully at 1732522779723 (+24 ms) 2024-11-25T08:19:39,725 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T08:19:39,757 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b210b1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=5eb3d201e8c9/172.17.0.2:0 2024-11-25T08:19:39,823 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T08:19:39,844 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T08:19:39,844 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T08:19:39,848 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T08:19:39,850 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T08:19:39,858 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-25T08:19:39,858 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T08:19:39,893 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T08:19:39,905 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-25T08:19:39,974 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-25T08:19:39,978 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T08:19:39,980 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-25T08:19:40,043 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-25T08:19:40,047 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T08:19:40,052 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-25T08:19:40,112 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-25T08:19:40,114 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-25T08:19:40,174 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-25T08:19:40,194 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T08:19:40,235 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-25T08:19:40,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-25T08:19:40,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-25T08:19:40,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:40,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:40,285 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5eb3d201e8c9,44851,1732522775045, sessionid=0x1014e35793c0000, setting cluster-up flag (Was=false) 2024-11-25T08:19:40,292 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-25T08:19:40,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:40,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:40,512 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-25T08:19:40,515 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:40,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:40,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:40,758 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-25T08:19:40,760 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:40,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:40,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:40,966 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-25T08:19:40,969 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:40,976 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T08:19:41,053 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T08:19:41,060 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(746): ClusterId : 20ef2d0a-a99b-4ad7-a633-df33eb0156be 2024-11-25T08:19:41,061 INFO [RS:0;5eb3d201e8c9:43089 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-25T08:19:41,063 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T08:19:41,063 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T08:19:41,070 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T08:19:41,076 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5eb3d201e8c9,44851,1732522775045 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T08:19:41,092 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T08:19:41,092 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-25T08:19:41,092 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5eb3d201e8c9:0, corePoolSize=5, maxPoolSize=5 2024-11-25T08:19:41,092 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5eb3d201e8c9:0, corePoolSize=5, maxPoolSize=5 2024-11-25T08:19:41,093 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=5, maxPoolSize=5 2024-11-25T08:19:41,093 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=5, maxPoolSize=5 2024-11-25T08:19:41,093 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5eb3d201e8c9:0, corePoolSize=10, maxPoolSize=10 2024-11-25T08:19:41,093 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,093 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=2, maxPoolSize=2 2024-11-25T08:19:41,093 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,098 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732522811098 2024-11-25T08:19:41,098 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T08:19:41,099 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T08:19:41,100 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T08:19:41,101 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T08:19:41,105 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:41,105 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T08:19:41,105 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-25T08:19:41,105 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T08:19:41,106 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T08:19:41,106 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T08:19:41,106 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T08:19:41,107 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,111 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T08:19:41,112 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T08:19:41,112 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T08:19:41,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741831_1007 (size=1321) 2024-11-25T08:19:41,124 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ced5b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:41,124 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:19:41,136 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:19:41,144 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-25T08:19:41,144 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T08:19:41,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:19:41,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:19:41,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:19:41,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a25a64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:41,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:19:41,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:19:41,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:41,200 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T08:19:41,200 DEBUG [RS:0;5eb3d201e8c9:43089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d952399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=5eb3d201e8c9/172.17.0.2:0 2024-11-25T08:19:41,204 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:19:41,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d336783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:41,215 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-25T08:19:41,217 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5eb3d201e8c9:43089 2024-11-25T08:19:41,218 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T08:19:41,218 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T08:19:41,220 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T08:19:41,220 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T08:19:41,220 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T08:19:41,220 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.large.0-1732522781220,5,FailOnTimeoutGroup] 2024-11-25T08:19:41,221 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.small.0-1732522781221,5,FailOnTimeoutGroup] 2024-11-25T08:19:41,221 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,221 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T08:19:41,223 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(2659): reportForDuty to master=5eb3d201e8c9,44851,1732522775045 with port=43089, startcode=1732522777179 2024-11-25T08:19:41,223 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,223 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,225 DEBUG [RS:0;5eb3d201e8c9:43089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T08:19:41,228 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42391, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T08:19:41,229 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44851 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:41,232 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44851 {}] master.ServerManager(517): Registering regionserver=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:41,243 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:19:41,243 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42243 2024-11-25T08:19:41,243 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=43797 2024-11-25T08:19:41,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-25T08:19:41,356 DEBUG [RS:0;5eb3d201e8c9:43089 {}] zookeeper.ZKUtil(111): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rs/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:41,356 WARN [RS:0;5eb3d201e8c9:43089 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T08:19:41,356 INFO [RS:0;5eb3d201e8c9:43089 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-25T08:19:41,356 INFO [RS:0;5eb3d201e8c9:43089 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-25T08:19:41,357 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:41,359 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5eb3d201e8c9,43089,1732522777179] 2024-11-25T08:19:41,382 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T08:19:41,394 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T08:19:41,399 INFO [RS:0;5eb3d201e8c9:43089 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T08:19:41,399 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,400 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T08:19:41,406 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T08:19:41,408 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,408 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,408 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,408 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,408 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,408 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,409 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5eb3d201e8c9:0, corePoolSize=2, maxPoolSize=2 2024-11-25T08:19:41,409 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,409 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,409 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,409 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,409 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,409 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5eb3d201e8c9:0, corePoolSize=1, maxPoolSize=1 2024-11-25T08:19:41,409 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0, corePoolSize=3, maxPoolSize=3 2024-11-25T08:19:41,410 DEBUG [RS:0;5eb3d201e8c9:43089 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5eb3d201e8c9:0, corePoolSize=3, maxPoolSize=3 2024-11-25T08:19:41,410 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,411 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,411 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,411 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,411 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,411 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,43089,1732522777179-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T08:19:41,429 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T08:19:41,431 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,43089,1732522777179-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,431 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,431 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.Replication(171): 5eb3d201e8c9,43089,1732522777179 started 2024-11-25T08:19:41,449 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:41,449 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1482): Serving as 5eb3d201e8c9,43089,1732522777179, RpcServer on 5eb3d201e8c9/172.17.0.2:43089, sessionid=0x1014e35793c0001 2024-11-25T08:19:41,450 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T08:19:41,450 DEBUG [RS:0;5eb3d201e8c9:43089 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:41,450 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,43089,1732522777179' 2024-11-25T08:19:41,450 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-25T08:19:41,451 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-25T08:19:41,452 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T08:19:41,452 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-25T08:19:41,452 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,43089,1732522777179' 2024-11-25T08:19:41,452 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:19:41,453 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:19:41,453 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-25T08:19:41,453 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-25T08:19:41,453 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T08:19:41,454 DEBUG [RS:0;5eb3d201e8c9:43089 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:41,454 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5eb3d201e8c9,43089,1732522777179' 2024-11-25T08:19:41,454 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-25T08:19:41,454 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-25T08:19:41,455 DEBUG [RS:0;5eb3d201e8c9:43089 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T08:19:41,455 INFO [RS:0;5eb3d201e8c9:43089 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T08:19:41,455 INFO [RS:0;5eb3d201e8c9:43089 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T08:19:41,518 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T08:19:41,519 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:19:41,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741832_1008 (size=32) 2024-11-25T08:19:41,560 INFO [RS:0;5eb3d201e8c9:43089 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T08:19:41,561 INFO [RS:0;5eb3d201e8c9:43089 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 5eb3d201e8c9,43089,1732522777179.regiongroup-0 2024-11-25T08:19:41,564 INFO [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0, suffix=, logDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179, archiveDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs, maxLogs=32 2024-11-25T08:19:41,580 DEBUG [RS:0;5eb3d201e8c9:43089 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522781566, exclude list is [], retry=0 2024-11-25T08:19:41,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:19:41,588 INFO [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522781566 2024-11-25T08:19:41,589 DEBUG [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:19:41,931 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:41,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T08:19:41,936 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T08:19:41,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:41,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:41,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T08:19:41,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T08:19:41,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:41,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:41,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T08:19:41,943 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T08:19:41,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:41,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:41,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T08:19:41,946 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T08:19:41,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:41,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:41,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T08:19:41,948 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740 2024-11-25T08:19:41,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740 2024-11-25T08:19:41,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T08:19:41,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T08:19:41,952 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T08:19:41,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T08:19:41,958 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T08:19:41,959 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9478456800, jitterRate=-0.11724992096424103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T08:19:41,961 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732522781931Initializing all the Stores at 1732522781933 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732522781933Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732522781933Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522781933Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732522781933Cleaning up temporary data from old regions at 1732522781952 (+19 ms)Region opened successfully at 1732522781961 (+9 ms) 2024-11-25T08:19:41,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T08:19:41,962 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T08:19:41,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T08:19:41,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T08:19:41,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T08:19:41,963 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T08:19:41,963 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732522781962Disabling compacts and flushes for region at 1732522781962Disabling writes for close at 1732522781962Writing region close event to WAL at 1732522781963 (+1 ms)Closed at 1732522781963 2024-11-25T08:19:41,968 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T08:19:41,968 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T08:19:41,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T08:19:41,986 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T08:19:41,989 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T08:19:42,142 DEBUG [5eb3d201e8c9:44851 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T08:19:42,156 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:42,163 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5eb3d201e8c9,43089,1732522777179, state=OPENING 2024-11-25T08:19:42,227 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T08:19:42,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:42,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:19:42,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-25T08:19:42,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-25T08:19:42,284 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T08:19:42,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:42,461 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T08:19:42,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50051, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T08:19:42,473 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T08:19:42,474 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-25T08:19:42,474 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-25T08:19:42,474 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T08:19:42,475 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T08:19:42,477 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C43089%2C1732522777179.meta, suffix=.meta, logDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179, archiveDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs, maxLogs=32 2024-11-25T08:19:42,493 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522782479.meta, exclude list is [], retry=0 2024-11-25T08:19:42,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:19:42,500 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522782479.meta 2024-11-25T08:19:42,500 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:19:42,500 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:42,506 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:19:42,507 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T08:19:42,511 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T08:19:42,511 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T08:19:42,515 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T08:19:42,516 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:42,516 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T08:19:42,516 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T08:19:42,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T08:19:42,520 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T08:19:42,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:42,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:42,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T08:19:42,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T08:19:42,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:42,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:42,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T08:19:42,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T08:19:42,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:42,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:42,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T08:19:42,528 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T08:19:42,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:42,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T08:19:42,529 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T08:19:42,530 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740 2024-11-25T08:19:42,533 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740 2024-11-25T08:19:42,536 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T08:19:42,536 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T08:19:42,537 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T08:19:42,540 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T08:19:42,542 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11767257280, jitterRate=0.09591123461723328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T08:19:42,542 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T08:19:42,543 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732522782517Writing region info on filesystem at 1732522782517Initializing all the Stores at 1732522782518 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732522782519 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732522782519Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522782519Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732522782519Cleaning up temporary data from old regions at 1732522782536 (+17 ms)Running coprocessor post-open hooks at 1732522782542 (+6 ms)Region opened successfully at 1732522782543 (+1 ms) 2024-11-25T08:19:42,550 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732522782452 2024-11-25T08:19:42,562 DEBUG [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T08:19:42,563 INFO [RS_OPEN_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T08:19:42,564 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:42,566 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5eb3d201e8c9,43089,1732522777179, state=OPEN 2024-11-25T08:19:42,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-25T08:19:42,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-25T08:19:42,651 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-25T08:19:42,651 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-25T08:19:42,651 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:42,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T08:19:42,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5eb3d201e8c9,43089,1732522777179 in 365 msec 2024-11-25T08:19:42,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T08:19:42,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 684 msec 2024-11-25T08:19:42,673 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T08:19:42,673 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T08:19:42,694 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:19:42,696 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:19:42,726 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:19:42,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47783, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:19:42,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7510 sec 2024-11-25T08:19:42,760 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732522782760, completionTime=-1 2024-11-25T08:19:42,765 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T08:19:42,765 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T08:19:42,803 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T08:19:42,803 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732522842803 2024-11-25T08:19:42,803 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732522902803 2024-11-25T08:19:42,803 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-11-25T08:19:42,806 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,44851,1732522775045-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:42,806 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,44851,1732522775045-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:42,806 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,44851,1732522775045-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:42,808 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5eb3d201e8c9:44851, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:42,809 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:42,809 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:42,820 DEBUG [master/5eb3d201e8c9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T08:19:42,847 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 5.311sec 2024-11-25T08:19:42,848 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T08:19:42,849 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T08:19:42,850 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T08:19:42,851 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T08:19:42,851 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T08:19:42,852 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,44851,1732522775045-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T08:19:42,853 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,44851,1732522775045-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T08:19:42,862 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T08:19:42,863 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T08:19:42,864 INFO [master/5eb3d201e8c9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5eb3d201e8c9,44851,1732522775045-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T08:19:42,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c2a639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:42,867 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T08:19:42,867 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T08:19:42,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:19:42,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:19:42,880 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:19:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:19:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:19:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794a94a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:42,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:19:42,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:19:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:42,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:19:42,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@511c9f94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:42,889 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:19:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:19:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:19:42,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:19:42,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:42,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-25T08:19:42,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/test.cache.data in system properties and HBase conf 2024-11-25T08:19:42,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T08:19:42,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir in system properties and HBase conf 2024-11-25T08:19:42,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T08:19:42,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T08:19:42,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T08:19:42,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T08:19:42,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T08:19:42,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T08:19:42,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T08:19:42,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/nfs.dump.dir in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/java.io.tmpdir in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T08:19:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T08:19:43,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741835_1011 (size=592039) 2024-11-25T08:19:43,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741836_1012 (size=1663647) 2024-11-25T08:19:44,713 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:44,816 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:45,071 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T08:19:45,071 WARN [Thread-251 {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-25T08:19:45,073 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T08:19:45,076 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T08:19:45,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T08:19:45,077 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T08:19:45,083 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:45,094 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T08:19:45,094 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T08:19:45,094 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T08:19:45,098 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fa5f4d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,AVAILABLE} 2024-11-25T08:19:45,098 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b1ead7d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,AVAILABLE} 2024-11-25T08:19:45,098 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@234d7a74{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-25T08:19:45,098 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fa2e322{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-25T08:19:45,274 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2024-11-25T08:19:45,274 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2024-11-25T08:19:45,274 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-25T08:19:45,276 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-25T08:19:45,319 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-25T08:19:45,363 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-25T08:19:45,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T08:19:45,491 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T08:19:45,501 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:19:45,501 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-25T08:19:45,502 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T08:19:45,502 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T08:19:45,502 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T08:19:45,502 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T08:19:45,572 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2024-11-25T08:19:45,590 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@14baf7c7{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/java.io.tmpdir/jetty-localhost-35921-hadoop-yarn-common-3_4_1_jar-_-any-809000613268934500/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-25T08:19:45,590 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3bcbb650{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/java.io.tmpdir/jetty-localhost-33885-hadoop-yarn-common-3_4_1_jar-_-any-17959331195606851788/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-25T08:19:45,591 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@20db1e36{HTTP/1.1, (http/1.1)}{localhost:33885} 2024-11-25T08:19:45,591 INFO [Time-limited test {}] server.Server(415): Started @19730ms 2024-11-25T08:19:45,591 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@54ee21da{HTTP/1.1, (http/1.1)}{localhost:35921} 2024-11-25T08:19:45,592 INFO [Thread-251 {}] server.Server(415): Started @19730ms 2024-11-25T08:19:45,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741837_1013 (size=5) 2024-11-25T08:19:46,986 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-25T08:19:46,990 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:47,021 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-25T08:19:47,022 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T08:19:47,029 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T08:19:47,029 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T08:19:47,029 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T08:19:47,030 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:47,031 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48e39004{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,AVAILABLE} 2024-11-25T08:19:47,031 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@27083e15{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-25T08:19:47,109 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-25T08:19:47,110 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-25T08:19:47,110 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-25T08:19:47,110 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-25T08:19:47,123 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-25T08:19:47,133 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-25T08:19:47,267 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-25T08:19:47,281 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40a758a4{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/java.io.tmpdir/jetty-localhost-33705-hadoop-yarn-common-3_4_1_jar-_-any-16244057316128844741/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-25T08:19:47,282 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4a6d3ac6{HTTP/1.1, (http/1.1)}{localhost:33705} 2024-11-25T08:19:47,282 INFO [Time-limited test {}] server.Server(415): Started @21421ms 2024-11-25T08:19:47,376 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T08:19:47,431 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-25T08:19:47,434 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:47,446 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-25T08:19:47,447 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T08:19:47,452 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T08:19:47,452 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T08:19:47,452 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T08:19:47,453 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T08:19:47,455 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f552527{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,AVAILABLE} 2024-11-25T08:19:47,455 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fdfb8bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-25T08:19:47,509 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-25T08:19:47,509 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-25T08:19:47,509 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-25T08:19:47,509 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-25T08:19:47,518 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-25T08:19:47,523 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-25T08:19:47,621 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-25T08:19:47,626 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4611faf6{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/java.io.tmpdir/jetty-localhost-32943-hadoop-yarn-common-3_4_1_jar-_-any-15623308212534731794/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-25T08:19:47,627 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f98650{HTTP/1.1, (http/1.1)}{localhost:32943} 2024-11-25T08:19:47,627 INFO [Time-limited test {}] server.Server(415): Started @21766ms 2024-11-25T08:19:47,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-25T08:19:47,663 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:42243/backupUT 2024-11-25T08:19:47,668 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T08:19:47,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40678cd3 2024-11-25T08:19:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T08:19:47,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T08:19:47,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-11-25T08:19:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-25T08:19:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T08:19:47,708 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 20 msec 2024-11-25T08:19:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T08:19:47,807 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-25T08:19:47,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-11-25T08:19:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-25T08:19:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-25T08:19:47,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 13 msec 2024-11-25T08:19:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-25T08:19:47,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-25T08:19:47,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-11-25T08:19:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-25T08:19:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-25T08:19:47,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 11 msec 2024-11-25T08:19:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-25T08:19:48,037 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-25T08:19:48,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-11-25T08:19:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-25T08:19:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T08:19:48,046 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-25T08:19:48,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 14 msec 2024-11-25T08:19:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T08:19:48,146 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-25T08:19:48,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1732522787663', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:19:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1732522787663 2024-11-25T08:19:48,172 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1732522787663 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T08:19:48,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1732522787663" procId is: 8 2024-11-25T08:19:48,175 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:48,177 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1732522787663 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T08:19:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-25T08:19:48,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741838_1014 (size=367) 2024-11-25T08:19:48,192 INFO [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9c7bcb575c3e97a1ab0077b07fb801c5, NAME => 'test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1732522787663', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:19:48,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741839_1015 (size=53) 2024-11-25T08:19:48,202 DEBUG [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(898): Instantiated test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:48,203 DEBUG [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(1722): Closing 9c7bcb575c3e97a1ab0077b07fb801c5, disabling compactions & flushes 2024-11-25T08:19:48,203 INFO [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(1755): Closing region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:48,203 DEBUG [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:48,203 DEBUG [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. after waiting 0 ms 2024-11-25T08:19:48,203 DEBUG [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:48,203 INFO [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(1973): Closed test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:48,203 DEBUG [RegionOpenAndInit-test-1732522787663-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9c7bcb575c3e97a1ab0077b07fb801c5: Waiting for close lock at 1732522788203Disabling compacts and flushes for region at 1732522788203Disabling writes for close at 1732522788203Writing region close event to WAL at 1732522788203Closed at 1732522788203 2024-11-25T08:19:48,206 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1732522787663 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T08:19:48,213 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732522788206"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732522788206"}]},"ts":"1732522788206"} 2024-11-25T08:19:48,226 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T08:19:48,229 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1732522787663 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T08:19:48,234 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732522787663","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522788229"}]},"ts":"1732522788229"} 2024-11-25T08:19:48,238 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732522787663, state=ENABLING in hbase:meta 2024-11-25T08:19:48,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, ASSIGN}] 2024-11-25T08:19:48,242 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, ASSIGN 2024-11-25T08:19:48,244 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, ASSIGN; state=OFFLINE, location=5eb3d201e8c9,43089,1732522777179; forceNewPlan=false, retain=false 2024-11-25T08:19:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-25T08:19:48,396 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:48,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, ASSIGN because future has completed 2024-11-25T08:19:48,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-25T08:19:48,577 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:48,577 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T08:19:48,577 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor 5eb3d201e8c9,43089,1732522777179.regiongroup-1 2024-11-25T08:19:48,580 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1, suffix=, logDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179, archiveDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs, maxLogs=32 2024-11-25T08:19:48,597 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522788582, exclude list is [], retry=0 2024-11-25T08:19:48,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:19:48,604 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522788582 2024-11-25T08:19:48,605 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:19:48,605 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c7bcb575c3e97a1ab0077b07fb801c5, NAME => 'test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:48,605 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:19:48,606 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732522787663 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,606 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:48,606 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,606 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,608 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,610 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c7bcb575c3e97a1ab0077b07fb801c5 columnFamilyName f 2024-11-25T08:19:48,611 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:48,612 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(327): Store=9c7bcb575c3e97a1ab0077b07fb801c5/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:48,612 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,613 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,614 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,614 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,614 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,617 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,621 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T08:19:48,622 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 9c7bcb575c3e97a1ab0077b07fb801c5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11428269280, jitterRate=0.0643405169248581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T08:19:48,622 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:48,623 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 9c7bcb575c3e97a1ab0077b07fb801c5: Running coprocessor pre-open hook at 1732522788606Writing region info on filesystem at 1732522788606Initializing all the Stores at 1732522788608 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522788608Cleaning up temporary data from old regions at 1732522788614 (+6 ms)Running coprocessor post-open hooks at 1732522788622 (+8 ms)Region opened successfully at 1732522788623 (+1 ms) 2024-11-25T08:19:48,625 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., pid=10, masterSystemTime=1732522788554 2024-11-25T08:19:48,629 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:48,629 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:48,630 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:48,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:19:48,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-25T08:19:48,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 in 234 msec 2024-11-25T08:19:48,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-25T08:19:48,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, ASSIGN in 399 msec 2024-11-25T08:19:48,644 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1732522787663 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T08:19:48,644 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732522787663","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522788644"}]},"ts":"1732522788644"} 2024-11-25T08:19:48,647 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732522787663, state=ENABLED in hbase:meta 2024-11-25T08:19:48,649 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1732522787663 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T08:19:48,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1732522787663 in 487 msec 2024-11-25T08:19:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-25T08:19:48,807 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1732522787663 completed 2024-11-25T08:19:48,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ade41d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:48,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:19:48,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:19:48,810 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:19:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:19:48,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:19:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29cad2de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:19:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:19:48,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:48,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:19:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de9fc78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:48,819 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:19:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:19:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:19:48,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:19:48,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732522787663', row='row0', locateType=CURRENT is [region=test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=2] 2024-11-25T08:19:48,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43089 {}] regionserver.HRegion(8528): writing data to region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. with WAL disabled. Data may be lost in the event of a crash. 2024-11-25T08:19:48,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17325227876631', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:19:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17325227876631 2024-11-25T08:19:48,998 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17325227876631 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T08:19:48,998 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:48,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17325227876631" procId is: 11 2024-11-25T08:19:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T08:19:49,000 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17325227876631 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T08:19:49,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741841_1017 (size=364) 2024-11-25T08:19:49,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T08:19:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T08:19:49,416 INFO [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 74322db2b59a5db2ea28349c44aa1b80, NAME => 'ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17325227876631', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:19:49,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741842_1018 (size=50) 2024-11-25T08:19:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T08:19:49,825 DEBUG [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:49,825 DEBUG [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(1722): Closing 74322db2b59a5db2ea28349c44aa1b80, disabling compactions & flushes 2024-11-25T08:19:49,826 INFO [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:19:49,826 DEBUG [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:19:49,826 DEBUG [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. after waiting 0 ms 2024-11-25T08:19:49,826 DEBUG [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:19:49,826 INFO [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:19:49,826 DEBUG [RegionOpenAndInit-ns2:test-17325227876631-pool-0 {}] regionserver.HRegion(1676): Region close journal for 74322db2b59a5db2ea28349c44aa1b80: Waiting for close lock at 1732522789825Disabling compacts and flushes for region at 1732522789825Disabling writes for close at 1732522789826 (+1 ms)Writing region close event to WAL at 1732522789826Closed at 1732522789826 2024-11-25T08:19:49,828 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17325227876631 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T08:19:49,828 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732522789828"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732522789828"}]},"ts":"1732522789828"} 2024-11-25T08:19:49,831 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T08:19:49,832 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17325227876631 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T08:19:49,833 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17325227876631","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522789832"}]},"ts":"1732522789832"} 2024-11-25T08:19:49,835 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17325227876631, state=ENABLING in hbase:meta 2024-11-25T08:19:49,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17325227876631, region=74322db2b59a5db2ea28349c44aa1b80, ASSIGN}] 2024-11-25T08:19:49,837 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17325227876631, region=74322db2b59a5db2ea28349c44aa1b80, ASSIGN 2024-11-25T08:19:49,838 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17325227876631, region=74322db2b59a5db2ea28349c44aa1b80, ASSIGN; state=OFFLINE, location=5eb3d201e8c9,43089,1732522777179; forceNewPlan=false, retain=false 2024-11-25T08:19:49,989 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=74322db2b59a5db2ea28349c44aa1b80, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:49,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17325227876631, region=74322db2b59a5db2ea28349c44aa1b80, ASSIGN because future has completed 2024-11-25T08:19:49,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74322db2b59a5db2ea28349c44aa1b80, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T08:19:50,152 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:19:50,152 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 74322db2b59a5db2ea28349c44aa1b80, NAME => 'ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:50,152 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:19:50,153 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17325227876631 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,153 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:50,153 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,153 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,155 INFO [StoreOpener-74322db2b59a5db2ea28349c44aa1b80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,157 INFO [StoreOpener-74322db2b59a5db2ea28349c44aa1b80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 74322db2b59a5db2ea28349c44aa1b80 columnFamilyName f 2024-11-25T08:19:50,157 DEBUG [StoreOpener-74322db2b59a5db2ea28349c44aa1b80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:50,158 INFO [StoreOpener-74322db2b59a5db2ea28349c44aa1b80-1 {}] regionserver.HStore(327): Store=74322db2b59a5db2ea28349c44aa1b80/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:50,158 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,160 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,160 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,161 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,161 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,164 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,167 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T08:19:50,168 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 74322db2b59a5db2ea28349c44aa1b80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11352752000, jitterRate=0.057307422161102295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T08:19:50,168 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:19:50,169 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 74322db2b59a5db2ea28349c44aa1b80: Running coprocessor pre-open hook at 1732522790153Writing region info on filesystem at 1732522790153Initializing all the Stores at 1732522790155 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522790155Cleaning up temporary data from old regions at 1732522790161 (+6 ms)Running coprocessor post-open hooks at 1732522790168 (+7 ms)Region opened successfully at 1732522790169 (+1 ms) 2024-11-25T08:19:50,171 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80., pid=13, masterSystemTime=1732522790147 2024-11-25T08:19:50,174 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:19:50,174 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:19:50,175 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=74322db2b59a5db2ea28349c44aa1b80, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:50,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 74322db2b59a5db2ea28349c44aa1b80, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:19:50,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-25T08:19:50,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 74322db2b59a5db2ea28349c44aa1b80, server=5eb3d201e8c9,43089,1732522777179 in 187 msec 2024-11-25T08:19:50,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T08:19:50,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17325227876631, region=74322db2b59a5db2ea28349c44aa1b80, ASSIGN in 349 msec 2024-11-25T08:19:50,189 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17325227876631 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T08:19:50,189 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17325227876631","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522790189"}]},"ts":"1732522790189"} 2024-11-25T08:19:50,192 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17325227876631, state=ENABLED in hbase:meta 2024-11-25T08:19:50,194 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17325227876631 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T08:19:50,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17325227876631 in 1.1990 sec 2024-11-25T08:19:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T08:19:51,146 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17325227876631 completed 2024-11-25T08:19:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17325227876631', row='row0', locateType=CURRENT is [region=ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=2] 2024-11-25T08:19:51,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43089 {}] regionserver.HRegion(8528): writing data to region ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. with WAL disabled. Data may be lost in the event of a crash. 2024-11-25T08:19:51,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17325227876632', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:19:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17325227876632 2024-11-25T08:19:51,288 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17325227876632 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T08:19:51,288 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:51,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17325227876632" procId is: 14 2024-11-25T08:19:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-25T08:19:51,290 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17325227876632 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T08:19:51,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741843_1019 (size=390) 2024-11-25T08:19:51,300 INFO [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 53385123a893b8dd1a5dbd90b0513715, NAME => 'ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17325227876632', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:19:51,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741844_1020 (size=50) 2024-11-25T08:19:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-25T08:19:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-25T08:19:51,710 DEBUG [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:51,710 DEBUG [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(1722): Closing 53385123a893b8dd1a5dbd90b0513715, disabling compactions & flushes 2024-11-25T08:19:51,710 INFO [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:19:51,710 DEBUG [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:19:51,710 DEBUG [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. after waiting 0 ms 2024-11-25T08:19:51,710 DEBUG [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:19:51,710 INFO [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:19:51,710 DEBUG [RegionOpenAndInit-ns3:test-17325227876632-pool-0 {}] regionserver.HRegion(1676): Region close journal for 53385123a893b8dd1a5dbd90b0513715: Waiting for close lock at 1732522791710Disabling compacts and flushes for region at 1732522791710Disabling writes for close at 1732522791710Writing region close event to WAL at 1732522791710Closed at 1732522791710 2024-11-25T08:19:51,712 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17325227876632 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T08:19:51,713 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732522791712"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732522791712"}]},"ts":"1732522791712"} 2024-11-25T08:19:51,715 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T08:19:51,717 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17325227876632 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T08:19:51,717 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17325227876632","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522791717"}]},"ts":"1732522791717"} 2024-11-25T08:19:51,719 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17325227876632, state=ENABLING in hbase:meta 2024-11-25T08:19:51,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17325227876632, region=53385123a893b8dd1a5dbd90b0513715, ASSIGN}] 2024-11-25T08:19:51,721 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17325227876632, region=53385123a893b8dd1a5dbd90b0513715, ASSIGN 2024-11-25T08:19:51,722 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17325227876632, region=53385123a893b8dd1a5dbd90b0513715, ASSIGN; state=OFFLINE, location=5eb3d201e8c9,43089,1732522777179; forceNewPlan=false, retain=false 2024-11-25T08:19:51,873 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=53385123a893b8dd1a5dbd90b0513715, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:51,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17325227876632, region=53385123a893b8dd1a5dbd90b0513715, ASSIGN because future has completed 2024-11-25T08:19:51,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53385123a893b8dd1a5dbd90b0513715, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-25T08:19:52,036 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:19:52,036 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => 53385123a893b8dd1a5dbd90b0513715, NAME => 'ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:52,037 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:19:52,037 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17325227876632 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,037 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:52,037 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,037 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,040 INFO [StoreOpener-53385123a893b8dd1a5dbd90b0513715-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,042 INFO [StoreOpener-53385123a893b8dd1a5dbd90b0513715-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53385123a893b8dd1a5dbd90b0513715 columnFamilyName f 2024-11-25T08:19:52,042 DEBUG [StoreOpener-53385123a893b8dd1a5dbd90b0513715-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:52,043 INFO [StoreOpener-53385123a893b8dd1a5dbd90b0513715-1 {}] regionserver.HStore(327): Store=53385123a893b8dd1a5dbd90b0513715/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:52,043 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,045 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns3/test-17325227876632/53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,046 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns3/test-17325227876632/53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,046 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,046 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,050 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,053 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns3/test-17325227876632/53385123a893b8dd1a5dbd90b0513715/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T08:19:52,054 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened 53385123a893b8dd1a5dbd90b0513715; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12038795360, jitterRate=0.12120018899440765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T08:19:52,054 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:19:52,091 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for 53385123a893b8dd1a5dbd90b0513715: Running coprocessor pre-open hook at 1732522792038Writing region info on filesystem at 1732522792038Initializing all the Stores at 1732522792039 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522792039Cleaning up temporary data from old regions at 1732522792046 (+7 ms)Running coprocessor post-open hooks at 1732522792054 (+8 ms)Region opened successfully at 1732522792090 (+36 ms) 2024-11-25T08:19:52,092 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715., pid=16, masterSystemTime=1732522792030 2024-11-25T08:19:52,095 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:19:52,095 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:19:52,096 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=53385123a893b8dd1a5dbd90b0513715, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:52,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53385123a893b8dd1a5dbd90b0513715, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:19:52,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-25T08:19:52,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 53385123a893b8dd1a5dbd90b0513715, server=5eb3d201e8c9,43089,1732522777179 in 224 msec 2024-11-25T08:19:52,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-25T08:19:52,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17325227876632, region=53385123a893b8dd1a5dbd90b0513715, ASSIGN in 386 msec 2024-11-25T08:19:52,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17325227876632 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T08:19:52,109 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17325227876632","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522792109"}]},"ts":"1732522792109"} 2024-11-25T08:19:52,112 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17325227876632, state=ENABLED in hbase:meta 2024-11-25T08:19:52,113 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17325227876632 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T08:19:52,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17325227876632 in 828 msec 2024-11-25T08:19:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-25T08:19:52,427 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17325227876632 completed 2024-11-25T08:19:52,427 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17325227876632 get assigned. Timeout = 60000ms 2024-11-25T08:19:52,429 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T08:19:52,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17325227876632 assigned to meta. Checking AM states. 2024-11-25T08:19:52,436 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T08:19:52,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17325227876632 assigned. 2024-11-25T08:19:52,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17325227876633', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:19:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17325227876633 2024-11-25T08:19:52,442 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17325227876633 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T08:19:52,442 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:52,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17325227876633" procId is: 17 2024-11-25T08:19:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T08:19:52,444 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17325227876633 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T08:19:52,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741845_1021 (size=390) 2024-11-25T08:19:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T08:19:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T08:19:52,858 INFO [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 44a4d8f1b8ed294e1ddee3eb2e0fe2e9, NAME => 'ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17325227876633', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:19:52,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741846_1022 (size=50) 2024-11-25T08:19:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T08:19:53,269 DEBUG [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:53,269 DEBUG [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(1722): Closing 44a4d8f1b8ed294e1ddee3eb2e0fe2e9, disabling compactions & flushes 2024-11-25T08:19:53,269 INFO [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:19:53,269 DEBUG [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:19:53,269 DEBUG [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. after waiting 0 ms 2024-11-25T08:19:53,269 DEBUG [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:19:53,269 INFO [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:19:53,269 DEBUG [RegionOpenAndInit-ns4:test-17325227876633-pool-0 {}] regionserver.HRegion(1676): Region close journal for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9: Waiting for close lock at 1732522793269Disabling compacts and flushes for region at 1732522793269Disabling writes for close at 1732522793269Writing region close event to WAL at 1732522793269Closed at 1732522793269 2024-11-25T08:19:53,272 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17325227876633 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T08:19:53,272 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732522793272"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732522793272"}]},"ts":"1732522793272"} 2024-11-25T08:19:53,275 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T08:19:53,276 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17325227876633 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T08:19:53,277 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17325227876633","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522793276"}]},"ts":"1732522793276"} 2024-11-25T08:19:53,279 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17325227876633, state=ENABLING in hbase:meta 2024-11-25T08:19:53,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17325227876633, region=44a4d8f1b8ed294e1ddee3eb2e0fe2e9, ASSIGN}] 2024-11-25T08:19:53,281 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17325227876633, region=44a4d8f1b8ed294e1ddee3eb2e0fe2e9, ASSIGN 2024-11-25T08:19:53,283 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17325227876633, region=44a4d8f1b8ed294e1ddee3eb2e0fe2e9, ASSIGN; state=OFFLINE, location=5eb3d201e8c9,43089,1732522777179; forceNewPlan=false, retain=false 2024-11-25T08:19:53,379 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17325227876631' 2024-11-25T08:19:53,380 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17325227876632' 2024-11-25T08:19:53,381 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1732522787663' 2024-11-25T08:19:53,433 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=44a4d8f1b8ed294e1ddee3eb2e0fe2e9, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:53,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17325227876633, region=44a4d8f1b8ed294e1ddee3eb2e0fe2e9, ASSIGN because future has completed 2024-11-25T08:19:53,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44a4d8f1b8ed294e1ddee3eb2e0fe2e9, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T08:19:53,596 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:19:53,596 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => 44a4d8f1b8ed294e1ddee3eb2e0fe2e9, NAME => 'ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:53,596 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:19:53,597 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17325227876633 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,597 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:53,597 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,597 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,599 INFO [StoreOpener-44a4d8f1b8ed294e1ddee3eb2e0fe2e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,602 INFO [StoreOpener-44a4d8f1b8ed294e1ddee3eb2e0fe2e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 columnFamilyName f 2024-11-25T08:19:53,602 DEBUG [StoreOpener-44a4d8f1b8ed294e1ddee3eb2e0fe2e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:53,603 INFO [StoreOpener-44a4d8f1b8ed294e1ddee3eb2e0fe2e9-1 {}] regionserver.HStore(327): Store=44a4d8f1b8ed294e1ddee3eb2e0fe2e9/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:53,603 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,604 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns4/test-17325227876633/44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,605 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns4/test-17325227876633/44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,605 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,605 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,608 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,613 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns4/test-17325227876633/44a4d8f1b8ed294e1ddee3eb2e0fe2e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T08:19:53,613 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened 44a4d8f1b8ed294e1ddee3eb2e0fe2e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11432575520, jitterRate=0.06474156677722931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T08:19:53,614 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:19:53,614 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9: Running coprocessor pre-open hook at 1732522793597Writing region info on filesystem at 1732522793597Initializing all the Stores at 1732522793598 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522793598Cleaning up temporary data from old regions at 1732522793605 (+7 ms)Running coprocessor post-open hooks at 1732522793614 (+9 ms)Region opened successfully at 1732522793614 2024-11-25T08:19:53,616 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9., pid=19, masterSystemTime=1732522793590 2024-11-25T08:19:53,619 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:19:53,619 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:19:53,620 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=44a4d8f1b8ed294e1ddee3eb2e0fe2e9, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:53,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44a4d8f1b8ed294e1ddee3eb2e0fe2e9, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:19:53,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-25T08:19:53,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 44a4d8f1b8ed294e1ddee3eb2e0fe2e9, server=5eb3d201e8c9,43089,1732522777179 in 187 msec 2024-11-25T08:19:53,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T08:19:53,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17325227876633, region=44a4d8f1b8ed294e1ddee3eb2e0fe2e9, ASSIGN in 348 msec 2024-11-25T08:19:53,631 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17325227876633 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T08:19:53,631 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17325227876633","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522793631"}]},"ts":"1732522793631"} 2024-11-25T08:19:53,634 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17325227876633, state=ENABLED in hbase:meta 2024-11-25T08:19:53,635 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17325227876633 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T08:19:53,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17325227876633 in 1.1960 sec 2024-11-25T08:19:54,109 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-25T08:19:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T08:19:54,596 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17325227876633 completed 2024-11-25T08:19:54,596 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17325227876633 get assigned. Timeout = 60000ms 2024-11-25T08:19:54,597 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T08:19:54,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17325227876633 assigned to meta. Checking AM states. 2024-11-25T08:19:54,602 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T08:19:54,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17325227876633 assigned. 2024-11-25T08:19:54,603 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T08:19:54,603 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:19:54,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:54,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:54,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:19:54,636 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=631, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=15, AvailableMemoryMB=5916 2024-11-25T08:19:54,637 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-11-25T08:19:54,639 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-25T08:19:54,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732522787663 from 'test-1732522787663', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732522787663', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:19:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732522787663 2024-11-25T08:19:54,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741847_1023 (size=908) 2024-11-25T08:19:55,073 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/.tabledesc/.tableinfo.0000000001.367 2024-11-25T08:19:55,074 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/.tabledesc/.tableinfo.0000000002.908 2024-11-25T08:19:55,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732522787663}] 2024-11-25T08:19:55,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE}] 2024-11-25T08:19:55,088 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE 2024-11-25T08:19:55,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=CLOSING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:55,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE because future has completed 2024-11-25T08:19:55,093 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T08:19:55,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:55,251 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:55,251 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T08:19:55,252 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 9c7bcb575c3e97a1ab0077b07fb801c5, disabling compactions & flushes 2024-11-25T08:19:55,252 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:55,252 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:55,252 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. after waiting 0 ms 2024-11-25T08:19:55,252 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:55,256 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 9c7bcb575c3e97a1ab0077b07fb801c5 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-25T08:19:55,326 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f/a259715a07084e5eaab5cf0e2878b34b is 37, key is row10/f:q1/1732522788851/Put/seqid=0 2024-11-25T08:19:55,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741848_1024 (size=8413) 2024-11-25T08:19:55,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17325227876631 2024-11-25T08:19:55,490 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17325227876631 Metrics about Tables on a single HBase RegionServer 2024-11-25T08:19:55,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17325227876633 2024-11-25T08:19:55,491 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17325227876633 Metrics about Tables on a single HBase RegionServer 2024-11-25T08:19:55,492 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732522787663 2024-11-25T08:19:55,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1732522787663 Metrics about Tables on a single HBase RegionServer 2024-11-25T08:19:55,493 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17325227876632 2024-11-25T08:19:55,493 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17325227876632 Metrics about Tables on a single HBase RegionServer 2024-11-25T08:19:55,738 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f/a259715a07084e5eaab5cf0e2878b34b 2024-11-25T08:19:55,805 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f/a259715a07084e5eaab5cf0e2878b34b as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b 2024-11-25T08:19:55,815 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b, entries=99, sequenceid=103, filesize=8.2 K 2024-11-25T08:19:55,822 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 9c7bcb575c3e97a1ab0077b07fb801c5 in 566ms, sequenceid=103, compaction requested=false 2024-11-25T08:19:55,830 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-25T08:19:55,834 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:19:55,834 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:55,834 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 9c7bcb575c3e97a1ab0077b07fb801c5: Waiting for close lock at 1732522795252Running coprocessor pre-close hooks at 1732522795252Disabling compacts and flushes for region at 1732522795252Disabling writes for close at 1732522795252Obtaining lock to block concurrent updates at 1732522795256 (+4 ms)Preparing flush snapshotting stores in 9c7bcb575c3e97a1ab0077b07fb801c5 at 1732522795256Finished memstore snapshotting test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1732522795265 (+9 ms)Flushing stores of test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. at 1732522795267 (+2 ms)Flushing 9c7bcb575c3e97a1ab0077b07fb801c5/f: creating writer at 1732522795269 (+2 ms)Flushing 9c7bcb575c3e97a1ab0077b07fb801c5/f: appending metadata at 1732522795314 (+45 ms)Flushing 9c7bcb575c3e97a1ab0077b07fb801c5/f: closing flushed file at 1732522795318 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@766bcc2f: reopening flushed file at 1732522795803 (+485 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 9c7bcb575c3e97a1ab0077b07fb801c5 in 566ms, sequenceid=103, compaction requested=false at 1732522795822 (+19 ms)Writing region close event to WAL at 1732522795824 (+2 ms)Running coprocessor post-close hooks at 1732522795831 (+7 ms)Closed at 1732522795834 (+3 ms) 2024-11-25T08:19:55,834 WARN [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 9c7bcb575c3e97a1ab0077b07fb801c5 to self. 2024-11-25T08:19:55,838 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:55,839 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=CLOSED 2024-11-25T08:19:55,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:19:55,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-25T08:19:55,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 in 750 msec 2024-11-25T08:19:55,847 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE; state=CLOSED, location=5eb3d201e8c9,43089,1732522777179; forceNewPlan=false, retain=true 2024-11-25T08:19:55,998 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:56,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE because future has completed 2024-11-25T08:19:56,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:56,161 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:56,161 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c7bcb575c3e97a1ab0077b07fb801c5, NAME => 'test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:56,161 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:19:56,162 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732522787663 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,162 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:56,162 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,162 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,164 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,165 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c7bcb575c3e97a1ab0077b07fb801c5 columnFamilyName f 2024-11-25T08:19:56,165 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:56,177 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b 2024-11-25T08:19:56,177 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(327): Store=9c7bcb575c3e97a1ab0077b07fb801c5/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:56,177 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,179 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c7bcb575c3e97a1ab0077b07fb801c5 columnFamilyName f3 2024-11-25T08:19:56,179 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:56,180 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(327): Store=9c7bcb575c3e97a1ab0077b07fb801c5/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:56,181 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,191 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c7bcb575c3e97a1ab0077b07fb801c5 columnFamilyName mob 2024-11-25T08:19:56,197 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:56,198 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(327): Store=9c7bcb575c3e97a1ab0077b07fb801c5/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:56,199 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,200 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,202 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,203 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,203 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,204 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732522787663 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T08:19:56,207 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,208 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 9c7bcb575c3e97a1ab0077b07fb801c5; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10333434240, jitterRate=-0.03762394189834595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T08:19:56,208 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:19:56,209 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 9c7bcb575c3e97a1ab0077b07fb801c5: Running coprocessor pre-open hook at 1732522796162Writing region info on filesystem at 1732522796162Initializing all the Stores at 1732522796164 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522796164Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522796164Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522796164Cleaning up temporary data from old regions at 1732522796203 (+39 ms)Running coprocessor post-open hooks at 1732522796208 (+5 ms)Region opened successfully at 1732522796209 (+1 ms) 2024-11-25T08:19:56,210 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., pid=24, masterSystemTime=1732522796155 2024-11-25T08:19:56,214 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:56,214 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:19:56,215 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=OPEN, openSeqNum=107, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:56,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:19:56,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-25T08:19:56,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 in 217 msec 2024-11-25T08:19:56,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-25T08:19:56,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE in 1.1350 sec 2024-11-25T08:19:56,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-25T08:19:56,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732522787663 in 1.1490 sec 2024-11-25T08:19:56,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732522787663 in 1.5840 sec 2024-11-25T08:19:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-25T08:19:56,238 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732522787663 completed 2024-11-25T08:19:56,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aab886f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:56,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:19:56,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:19:56,240 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:19:56,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:19:56,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:19:56,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44580fdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:56,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:19:56,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:19:56,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:56,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:19:56,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45885775, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:56,245 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:19:56,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:19:56,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:19:56,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:19:56,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732522787663', row='row-test-1732522787663-3-0', locateType=CURRENT is [region=test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=107] 2024-11-25T08:19:56,296 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T08:19:56,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5eb3d201e8c9,44851,1732522775045 2024-11-25T08:19:56,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36f280a6 2024-11-25T08:19:56,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T08:19:56,301 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T08:19:56,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-11-25T08:19:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-25T08:19:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T08:19:56,323 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 9 msec 2024-11-25T08:19:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T08:19:56,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-25T08:19:56,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:19:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-25T08:19:56,447 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T08:19:56,448 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:56,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-25T08:19:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T08:19:56,449 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T08:19:56,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741849_1025 (size=635) 2024-11-25T08:19:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T08:19:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T08:19:56,861 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66dbbb90362d3aeb47c6625cd227a909, NAME => 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:19:56,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741850_1026 (size=40) 2024-11-25T08:19:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T08:19:57,272 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:57,272 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 66dbbb90362d3aeb47c6625cd227a909, disabling compactions & flushes 2024-11-25T08:19:57,272 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:19:57,272 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:19:57,272 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. after waiting 0 ms 2024-11-25T08:19:57,272 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:19:57,273 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:19:57,273 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66dbbb90362d3aeb47c6625cd227a909: Waiting for close lock at 1732522797272Disabling compacts and flushes for region at 1732522797272Disabling writes for close at 1732522797272Writing region close event to WAL at 1732522797272Closed at 1732522797272 2024-11-25T08:19:57,275 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T08:19:57,275 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732522797275"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732522797275"}]},"ts":"1732522797275"} 2024-11-25T08:19:57,278 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T08:19:57,280 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T08:19:57,280 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522797280"}]},"ts":"1732522797280"} 2024-11-25T08:19:57,282 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-25T08:19:57,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN}] 2024-11-25T08:19:57,284 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN 2024-11-25T08:19:57,285 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN; state=OFFLINE, location=5eb3d201e8c9,43089,1732522777179; forceNewPlan=false, retain=false 2024-11-25T08:19:57,436 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=66dbbb90362d3aeb47c6625cd227a909, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:57,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN because future has completed 2024-11-25T08:19:57,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T08:19:57,597 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:19:57,597 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 66dbbb90362d3aeb47c6625cd227a909, NAME => 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:57,598 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:19:57,598 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,598 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:57,598 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,598 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,600 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,603 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66dbbb90362d3aeb47c6625cd227a909 columnFamilyName meta 2024-11-25T08:19:57,603 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:57,603 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.HStore(327): Store=66dbbb90362d3aeb47c6625cd227a909/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:57,604 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,605 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66dbbb90362d3aeb47c6625cd227a909 columnFamilyName session 2024-11-25T08:19:57,606 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:57,606 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.HStore(327): Store=66dbbb90362d3aeb47c6625cd227a909/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:57,606 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,607 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,608 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,609 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,609 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,610 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T08:19:57,612 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,615 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T08:19:57,616 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 66dbbb90362d3aeb47c6625cd227a909; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9691763200, jitterRate=-0.09738421440124512}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T08:19:57,616 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:19:57,617 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 66dbbb90362d3aeb47c6625cd227a909: Running coprocessor pre-open hook at 1732522797598Writing region info on filesystem at 1732522797598Initializing all the Stores at 1732522797600 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522797600Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522797600Cleaning up temporary data from old regions at 1732522797609 (+9 ms)Running coprocessor post-open hooks at 1732522797616 (+7 ms)Region opened successfully at 1732522797617 (+1 ms) 2024-11-25T08:19:57,618 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., pid=28, masterSystemTime=1732522797592 2024-11-25T08:19:57,621 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:19:57,621 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:19:57,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=66dbbb90362d3aeb47c6625cd227a909, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:57,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:19:57,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-25T08:19:57,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179 in 187 msec 2024-11-25T08:19:57,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-25T08:19:57,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN in 346 msec 2024-11-25T08:19:57,634 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T08:19:57,634 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522797634"}]},"ts":"1732522797634"} 2024-11-25T08:19:57,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-25T08:19:57,639 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T08:19:57,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.1950 sec 2024-11-25T08:19:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-25T08:19:58,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-25T08:19:58,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:19:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-25T08:19:58,636 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T08:19:58,636 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:58,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-25T08:19:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T08:19:58,638 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T08:19:58,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741851_1027 (size=640) 2024-11-25T08:19:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T08:19:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T08:19:59,050 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => daf94d1506be6ea0f96237e4c58ae364, NAME => 'backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:19:59,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741852_1028 (size=45) 2024-11-25T08:19:59,059 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:59,059 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing daf94d1506be6ea0f96237e4c58ae364, disabling compactions & flushes 2024-11-25T08:19:59,060 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:19:59,060 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:19:59,060 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. after waiting 0 ms 2024-11-25T08:19:59,060 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:19:59,060 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:19:59,060 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for daf94d1506be6ea0f96237e4c58ae364: Waiting for close lock at 1732522799059Disabling compacts and flushes for region at 1732522799059Disabling writes for close at 1732522799060 (+1 ms)Writing region close event to WAL at 1732522799060Closed at 1732522799060 2024-11-25T08:19:59,061 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T08:19:59,062 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732522799061"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732522799061"}]},"ts":"1732522799061"} 2024-11-25T08:19:59,065 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T08:19:59,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T08:19:59,067 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522799067"}]},"ts":"1732522799067"} 2024-11-25T08:19:59,069 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-25T08:19:59,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=daf94d1506be6ea0f96237e4c58ae364, ASSIGN}] 2024-11-25T08:19:59,071 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=daf94d1506be6ea0f96237e4c58ae364, ASSIGN 2024-11-25T08:19:59,072 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=daf94d1506be6ea0f96237e4c58ae364, ASSIGN; state=OFFLINE, location=5eb3d201e8c9,43089,1732522777179; forceNewPlan=false, retain=false 2024-11-25T08:19:59,099 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-25T08:19:59,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=daf94d1506be6ea0f96237e4c58ae364, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:59,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=daf94d1506be6ea0f96237e4c58ae364, ASSIGN because future has completed 2024-11-25T08:19:59,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure daf94d1506be6ea0f96237e4c58ae364, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:19:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T08:19:59,387 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17325227876633' 2024-11-25T08:19:59,388 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:19:59,388 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-25T08:19:59,388 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => daf94d1506be6ea0f96237e4c58ae364, NAME => 'backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:19:59,388 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:19:59,389 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,389 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:19:59,389 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,389 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,396 INFO [StoreOpener-daf94d1506be6ea0f96237e4c58ae364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,400 INFO [StoreOpener-daf94d1506be6ea0f96237e4c58ae364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region daf94d1506be6ea0f96237e4c58ae364 columnFamilyName meta 2024-11-25T08:19:59,400 DEBUG [StoreOpener-daf94d1506be6ea0f96237e4c58ae364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:59,401 INFO [StoreOpener-daf94d1506be6ea0f96237e4c58ae364-1 {}] regionserver.HStore(327): Store=daf94d1506be6ea0f96237e4c58ae364/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:59,401 INFO [StoreOpener-daf94d1506be6ea0f96237e4c58ae364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,403 INFO [StoreOpener-daf94d1506be6ea0f96237e4c58ae364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region daf94d1506be6ea0f96237e4c58ae364 columnFamilyName session 2024-11-25T08:19:59,403 DEBUG [StoreOpener-daf94d1506be6ea0f96237e4c58ae364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:19:59,404 INFO [StoreOpener-daf94d1506be6ea0f96237e4c58ae364-1 {}] regionserver.HStore(327): Store=daf94d1506be6ea0f96237e4c58ae364/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:19:59,404 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,405 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system_bulk/daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,405 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system_bulk/daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,407 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,407 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,407 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T08:19:59,409 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,413 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system_bulk/daf94d1506be6ea0f96237e4c58ae364/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T08:19:59,415 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened daf94d1506be6ea0f96237e4c58ae364; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11356806400, jitterRate=0.057685017585754395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T08:19:59,415 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:19:59,416 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for daf94d1506be6ea0f96237e4c58ae364: Running coprocessor pre-open hook at 1732522799389Writing region info on filesystem at 1732522799389Initializing all the Stores at 1732522799393 (+4 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522799393Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522799396 (+3 ms)Cleaning up temporary data from old regions at 1732522799407 (+11 ms)Running coprocessor post-open hooks at 1732522799415 (+8 ms)Region opened successfully at 1732522799416 (+1 ms) 2024-11-25T08:19:59,417 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364., pid=31, masterSystemTime=1732522799380 2024-11-25T08:19:59,420 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:19:59,420 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:19:59,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=daf94d1506be6ea0f96237e4c58ae364, regionState=OPEN, openSeqNum=2, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:19:59,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure daf94d1506be6ea0f96237e4c58ae364, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:19:59,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-25T08:19:59,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure daf94d1506be6ea0f96237e4c58ae364, server=5eb3d201e8c9,43089,1732522777179 in 199 msec 2024-11-25T08:19:59,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-25T08:19:59,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=daf94d1506be6ea0f96237e4c58ae364, ASSIGN in 360 msec 2024-11-25T08:19:59,434 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T08:19:59,434 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522799434"}]},"ts":"1732522799434"} 2024-11-25T08:19:59,437 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-25T08:19:59,438 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T08:19:59,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 805 msec 2024-11-25T08:19:59,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T08:19:59,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-25T08:19:59,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:19:59,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:19:59,796 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-25T08:19:59,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=2] 2024-11-25T08:19:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43089 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-25T08:19:59,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:19:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522799833 (current time:1732522799833). 2024-11-25T08:19:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:19:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-25T08:19:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e8661e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:19:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:19:59,836 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:19:59,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:19:59,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:19:59,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b1325c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:59,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:19:59,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:19:59,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:59,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:19:59,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47568b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:19:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:19:59,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:19:59,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:19:59,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:19:59,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:19:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:19:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:19:59,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:19:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:19:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:19:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-25T08:19:59,858 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:19:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-25T08:19:59,864 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:19:59,881 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:19:59,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741853_1029 (size=52) 2024-11-25T08:19:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-25T08:20:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-25T08:20:00,291 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:00,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909}] 2024-11-25T08:20:00,300 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:00,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-25T08:20:00,460 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:20:00,461 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 66dbbb90362d3aeb47c6625cd227a909 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-25T08:20:00,480 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/7fac197fb6374a9ea5dfe210eeb91f1b is 49, key is activesession:/session:c/1732522799818/Put/seqid=0 2024-11-25T08:20:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-25T08:20:00,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741854_1030 (size=5049) 2024-11-25T08:20:00,487 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/7fac197fb6374a9ea5dfe210eeb91f1b 2024-11-25T08:20:00,496 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/7fac197fb6374a9ea5dfe210eeb91f1b as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b 2024-11-25T08:20:00,503 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b, entries=1, sequenceid=5, filesize=4.9 K 2024-11-25T08:20:00,504 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 66dbbb90362d3aeb47c6625cd227a909 in 43ms, sequenceid=5, compaction requested=false 2024-11-25T08:20:00,505 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 66dbbb90362d3aeb47c6625cd227a909: 2024-11-25T08:20:00,505 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. for snapshot_backup_system completed. 2024-11-25T08:20:00,506 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.' region-info for snapshot=snapshot_backup_system 2024-11-25T08:20:00,510 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:00,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-25T08:20:00,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b] hfiles 2024-11-25T08:20:00,514 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b for snapshot=snapshot_backup_system 2024-11-25T08:20:00,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741855_1031 (size=95) 2024-11-25T08:20:00,930 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:20:00,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-25T08:20:00,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-25T08:20:00,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:00,936 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:00,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-25T08:20:00,940 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:00,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 in 643 msec 2024-11-25T08:20:00,942 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:00,943 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:00,943 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-25T08:20:00,946 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-25T08:20:00,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741856_1032 (size=731) 2024-11-25T08:20:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-25T08:20:01,368 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:01,382 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:01,382 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_backup_system 2024-11-25T08:20:01,385 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:01,385 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-25T08:20:01,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.5320 sec 2024-11-25T08:20:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-25T08:20:02,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-25T08:20:02,007 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732522796282 started at 1732522802007. 2024-11-25T08:20:02,024 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732522796282 has been started. 2024-11-25T08:20:02,030 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-25T08:20:02,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-25T08:20:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-25T08:20:02,040 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-25T08:20:02,040 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-25T08:20:02,040 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-25T08:20:02,041 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-25T08:20:02,042 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,042 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:02,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-25T08:20:02,074 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,075 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-25T08:20:02,075 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:20:02,075 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,075 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-25T08:20:02,076 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:02,076 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,077 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-25T08:20:02,077 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:02,077 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-25T08:20:02,079 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-25T08:20:02,079 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-25T08:20:02,080 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-25T08:20:02,080 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-25T08:20:02,080 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-25T08:20:02,080 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-25T08:20:02,080 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '5eb3d201e8c9,43089,1732522777179' joining acquired barrier for procedure (rolllog) in zk 2024-11-25T08:20:02,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,151 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:02,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:02,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:02,152 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-25T08:20:02,152 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-25T08:20:02,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:02,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:02,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:02,154 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,154 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:02,155 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '5eb3d201e8c9,43089,1732522777179' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-25T08:20:02,155 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6cabdd28[Count = 0] remaining members to acquire global barrier 2024-11-25T08:20:02,155 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-25T08:20:02,155 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:02,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:02,213 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:02,213 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:02,213 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-25T08:20:02,214 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,214 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-25T08:20:02,214 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-25T08:20:02,215 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,222 INFO [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732522781566 highest: 1732522788582 on 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,223 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.meta:.meta(num 1732522782479) roll requested 2024-11-25T08:20:02,244 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522802223.meta, exclude list is [], retry=0 2024-11-25T08:20:02,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:02,251 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522782479.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522802223.meta 2024-11-25T08:20:02,252 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:02,253 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522782479.meta is not closed yet, will try archiving it next time 2024-11-25T08:20:02,253 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1:(num 1732522788582) roll requested 2024-11-25T08:20:02,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741834_1010 (size=12055) 2024-11-25T08:20:02,278 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253, exclude list is [], retry=0 2024-11-25T08:20:02,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:02,285 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522788582 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 2024-11-25T08:20:02,286 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:02,286 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522788582 is not closed yet, will try archiving it next time 2024-11-25T08:20:02,286 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0:(num 1732522781566) roll requested 2024-11-25T08:20:02,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741840_1016 (size=4756) 2024-11-25T08:20:02,304 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286, exclude list is [], retry=0 2024-11-25T08:20:02,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:02,313 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522781566 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 2024-11-25T08:20:02,314 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:02,314 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522781566 is not closed yet, will try archiving it next time 2024-11-25T08:20:02,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741833_1009 (size=1034) 2024-11-25T08:20:02,321 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522781566 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522781566 2024-11-25T08:20:02,323 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 100 2024-11-25T08:20:02,324 INFO [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732522802286 on 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,324 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T08:20:02,324 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is 5eb3d201e8c9,44851,1732522775045 2024-11-25T08:20:02,324 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f25e62b 2024-11-25T08:20:02,325 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T08:20:02,326 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59187, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-25T08:20:02,327 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:02,327 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:02,328 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:02,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59413, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-25T08:20:02,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:02,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:02,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:42243/backupUT\x00', locateType=CURRENT is [region=backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=2] 2024-11-25T08:20:02,358 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-25T08:20:02,359 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '5eb3d201e8c9,43089,1732522777179' in zk 2024-11-25T08:20:02,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,413 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-25T08:20:02,413 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-25T08:20:02,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:02,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:02,413 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-25T08:20:02,415 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:02,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:02,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:02,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:02,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:02,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,420 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '5eb3d201e8c9,43089,1732522777179': 2024-11-25T08:20:02,420 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '5eb3d201e8c9,43089,1732522777179' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-25T08:20:02,421 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-25T08:20:02,421 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-25T08:20:02,421 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-25T08:20:02,421 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-25T08:20:02,421 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-25T08:20:02,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-25T08:20:02,444 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:02,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:02,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,445 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-25T08:20:02,445 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:20:02,445 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:02,446 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,446 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:02,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:02,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:02,449 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,450 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:02,450 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,451 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:02,451 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:02,497 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:02,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:02,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:02,497 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-25T08:20:02,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-25T08:20:02,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-25T08:20:02,498 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-25T08:20:02,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:20:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-25T08:20:02,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } 2024-11-25T08:20:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522802514 (current time:1732522802514). 2024-11-25T08:20:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732522802513_default_test-1732522787663 VERSION not specified, setting to 2 2024-11-25T08:20:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38bae65e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:02,516 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:02,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:02,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:02,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eb73cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:02,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:02,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:02,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:02,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:02,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1626da97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:02,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:02,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:02,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:02,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:02,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } 2024-11-25T08:20:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-25T08:20:02,526 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T08:20:02,527 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:02,530 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:02,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741860_1036 (size=84) 2024-11-25T08:20:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T08:20:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T08:20:02,939 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:02,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5}] 2024-11-25T08:20:02,940 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:03,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-25T08:20:03,093 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:03,094 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 9c7bcb575c3e97a1ab0077b07fb801c5 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-25T08:20:03,113 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f3/d4be02add9f447c3b5cf47114241bc27 is 58, key is row-test-1732522787663-3-0/f3:q1/1732522796253/Put/seqid=0 2024-11-25T08:20:03,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741861_1037 (size=5387) 2024-11-25T08:20:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T08:20:03,520 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f3/d4be02add9f447c3b5cf47114241bc27 2024-11-25T08:20:03,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741862_1038 (size=4469) 2024-11-25T08:20:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T08:20:03,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/mob/9b801d1d7dfa4281823e96a92c96fe02 is 59, key is row-test-1732522787663-3-0/mob:q1/1732522796267/Put/seqid=0 2024-11-25T08:20:03,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741863_1039 (size=5484) 2024-11-25T08:20:04,091 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T08:20:04,378 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/mob/9b801d1d7dfa4281823e96a92c96fe02 2024-11-25T08:20:04,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f3/d4be02add9f447c3b5cf47114241bc27 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27 2024-11-25T08:20:04,400 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27, entries=6, sequenceid=121, filesize=5.3 K 2024-11-25T08:20:04,402 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/mob/9b801d1d7dfa4281823e96a92c96fe02 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 2024-11-25T08:20:04,411 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02, entries=6, sequenceid=121, filesize=5.4 K 2024-11-25T08:20:04,413 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 9c7bcb575c3e97a1ab0077b07fb801c5 in 1320ms, sequenceid=121, compaction requested=false 2024-11-25T08:20:04,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 9c7bcb575c3e97a1ab0077b07fb801c5: 2024-11-25T08:20:04,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. for snapshot_1732522802513_default_test-1732522787663 completed. 2024-11-25T08:20:04,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.' region-info for snapshot=snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:04,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:04,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b] hfiles 2024-11-25T08:20:04,414 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b for snapshot=snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:04,414 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27] hfiles 2024-11-25T08:20:04,414 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27 for snapshot=snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:04,414 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02] hfiles 2024-11-25T08:20:04,414 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 for snapshot=snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:04,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741864_1040 (size=185) 2024-11-25T08:20:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T08:20:04,822 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:04,823 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-25T08:20:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-25T08:20:04,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732522802513_default_test-1732522787663 on region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:04,824 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:04,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-25T08:20:04,828 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:04,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5 in 1.8860 sec 2024-11-25T08:20:04,829 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:04,832 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-25T08:20:04,832 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-25T08:20:04,832 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:04,833 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-25T08:20:04,833 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:04,834 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-25T08:20:04,834 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:04,834 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-25T08:20:04,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741865_1041 (size=49) 2024-11-25T08:20:04,891 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-25T08:20:05,244 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:05,244 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:05,245 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:05,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741866_1042 (size=1146) 2024-11-25T08:20:05,393 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-25T08:20:05,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-25T08:20:05,490 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-25T08:20:05,490 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-25T08:20:05,491 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-25T08:20:05,661 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:05,671 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:05,672 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522802513_default_test-1732522787663 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:05,674 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:05,674 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-25T08:20:05,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732522802513_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } in 3.1500 sec 2024-11-25T08:20:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-25T08:20:06,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732522787663 completed 2024-11-25T08:20:06,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } 2024-11-25T08:20:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522806697 (current time:1732522806697). 2024-11-25T08:20:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732522806696_ns2_test-17325227876631 VERSION not specified, setting to 2 2024-11-25T08:20:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330dfe36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:06,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:06,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:06,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:06,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8656f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:06,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:06,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:06,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:06,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:06,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f4e1d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:06,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:06,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:06,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:06,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:06,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:06,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } 2024-11-25T08:20:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-25T08:20:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T08:20:06,710 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:06,711 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:06,715 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:06,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741867_1043 (size=86) 2024-11-25T08:20:06,723 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:06,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 74322db2b59a5db2ea28349c44aa1b80}] 2024-11-25T08:20:06,724 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:20:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T08:20:06,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-25T08:20:06,877 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:20:06,877 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 74322db2b59a5db2ea28349c44aa1b80 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-25T08:20:06,898 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/.tmp/f/4b485798ffaa4bf699dffe1c33406876 is 37, key is row10/f:q1/1732522791168/Put/seqid=0 2024-11-25T08:20:06,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741868_1044 (size=8413) 2024-11-25T08:20:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T08:20:07,303 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/.tmp/f/4b485798ffaa4bf699dffe1c33406876 2024-11-25T08:20:07,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/.tmp/f/4b485798ffaa4bf699dffe1c33406876 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876 2024-11-25T08:20:07,320 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876, entries=99, sequenceid=103, filesize=8.2 K 2024-11-25T08:20:07,321 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 74322db2b59a5db2ea28349c44aa1b80 in 444ms, sequenceid=103, compaction requested=false 2024-11-25T08:20:07,322 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 74322db2b59a5db2ea28349c44aa1b80: 2024-11-25T08:20:07,322 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. for snapshot_1732522806696_ns2_test-17325227876631 completed. 2024-11-25T08:20:07,322 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80.' region-info for snapshot=snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:07,322 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:07,322 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876] hfiles 2024-11-25T08:20:07,322 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876 for snapshot=snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:07,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741869_1045 (size=91) 2024-11-25T08:20:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T08:20:07,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:20:07,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-25T08:20:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-25T08:20:07,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732522806696_ns2_test-17325227876631 on region 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:20:07,731 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:20:07,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-25T08:20:07,734 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:07,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 74322db2b59a5db2ea28349c44aa1b80 in 1.0090 sec 2024-11-25T08:20:07,735 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:07,736 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:07,736 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:07,737 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:07,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741870_1046 (size=456) 2024-11-25T08:20:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T08:20:08,149 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:08,157 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:08,158 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522806696_ns2_test-17325227876631 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:08,160 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:08,160 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-25T08:20:08,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732522806696_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } in 1.4530 sec 2024-11-25T08:20:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-25T08:20:08,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17325227876631 completed 2024-11-25T08:20:08,857 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732522796282 2024-11-25T08:20:08,857 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-25T08:20:08,859 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-25T08:20:08,861 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732522796282_ns2:test-17325227876631 2024-11-25T08:20:08,862 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732522806696_ns2_test-17325227876631 to hdfs://localhost:42243/backupUT/backup_1732522796282/ns2/test-17325227876631/ 2024-11-25T08:20:08,869 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-25T08:20:08,909 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:42243, inputRoot=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:20:08,909 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1328517145_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42243/backupUT/backup_1732522796282/ns2/test-17325227876631, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42243/backupUT/backup_1732522796282/ns2/test-17325227876631/.hbase-snapshot/.tmp/snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:08,912 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-25T08:20:08,918 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522806696_ns2_test-17325227876631 to hdfs://localhost:42243/backupUT/backup_1732522796282/ns2/test-17325227876631/.hbase-snapshot/.tmp/snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:08,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741871_1047 (size=86) 2024-11-25T08:20:08,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741872_1048 (size=456) 2024-11-25T08:20:09,341 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:09,341 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:09,342 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,362 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-4519324391274181586.jar 2024-11-25T08:20:10,362 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,363 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,430 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-4623149229622275907.jar 2024-11-25T08:20:10,431 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,431 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,432 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,432 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,432 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,433 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:10,433 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:20:10,434 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:20:10,434 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:20:10,434 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:20:10,435 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:20:10,435 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:20:10,435 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:20:10,436 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:20:10,436 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:20:10,436 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:20:10,437 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:20:10,439 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:10,439 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:10,440 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:10,440 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:10,440 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:10,441 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:10,441 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:10,512 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:20:10,533 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:20:10,544 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732522806696_ns2_test-17325227876631' hfile list 2024-11-25T08:20:10,552 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-11-25T08:20:10,797 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local930258066_0001_c3136660-dfd3-4c80-9468-87951c6a00b5/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-25T08:20:10,797 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local930258066_0001_c3136660-dfd3-4c80-9468-87951c6a00b5/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-25T08:20:10,981 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-25T08:20:11,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741873_1049 (size=8413) 2024-11-25T08:20:11,490 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876 output=hdfs://localhost:42243/backupUT/backup_1732522796282/ns2/test-17325227876631/archive/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876 2024-11-25T08:20:11,492 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.016M/sec 2024-11-25T08:20:11,892 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-25T08:20:11,893 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-25T08:20:11,900 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:11,900 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732522806696_ns2_test-17325227876631 finished. 2024-11-25T08:20:11,900 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732522796282_test-1732522787663 2024-11-25T08:20:11,900 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732522802513_default_test-1732522787663 to hdfs://localhost:42243/backupUT/backup_1732522796282/default/test-1732522787663/ 2024-11-25T08:20:11,900 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-25T08:20:11,932 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:42243, inputRoot=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:20:11,932 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1328517145_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42243/backupUT/backup_1732522796282/default/test-1732522787663, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42243/backupUT/backup_1732522796282/default/test-1732522787663/.hbase-snapshot/.tmp/snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:11,934 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-25T08:20:11,942 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522802513_default_test-1732522787663 to hdfs://localhost:42243/backupUT/backup_1732522796282/default/test-1732522787663/.hbase-snapshot/.tmp/snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:11,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741875_1051 (size=1146) 2024-11-25T08:20:11,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741874_1050 (size=84) 2024-11-25T08:20:12,355 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:12,356 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:12,356 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,363 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-7953163762520994524.jar 2024-11-25T08:20:13,363 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,364 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,431 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-15798269425369002380.jar 2024-11-25T08:20:13,432 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,432 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,433 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,433 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,433 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,434 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:13,434 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:20:13,434 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:20:13,435 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:20:13,435 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:20:13,435 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:20:13,436 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:20:13,436 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:20:13,436 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:20:13,437 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:20:13,437 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:20:13,437 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:20:13,438 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:13,438 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:13,438 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:13,439 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:13,439 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:13,439 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:13,439 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:13,483 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:20:13,497 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:20:13,504 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732522802513_default_test-1732522787663' hfile list 2024-11-25T08:20:13,507 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-11-25T08:20:13,601 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local2133822155_0002_499a2cac-927a-4e07-b1aa-4ba183a86b2c/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-25T08:20:13,601 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local2133822155_0002_499a2cac-927a-4e07-b1aa-4ba183a86b2c/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-25T08:20:13,680 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-25T08:20:13,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741876_1052 (size=8413) 2024-11-25T08:20:14,126 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b output=hdfs://localhost:42243/backupUT/backup_1732522796282/default/test-1732522787663/archive/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b 2024-11-25T08:20:14,126 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-25T08:20:14,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741877_1053 (size=5484) 2024-11-25T08:20:14,601 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 output=hdfs://localhost:42243/backupUT/backup_1732522796282/default/test-1732522787663/archive/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 2024-11-25T08:20:14,601 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.011M/sec 2024-11-25T08:20:14,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741878_1054 (size=5387) 2024-11-25T08:20:15,071 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27 output=hdfs://localhost:42243/backupUT/backup_1732522796282/default/test-1732522787663/archive/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27 2024-11-25T08:20:15,071 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-11-25T08:20:15,648 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-25T08:20:15,649 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-25T08:20:15,658 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:15,658 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732522802513_default_test-1732522787663 finished. 2024-11-25T08:20:15,658 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17325227876631 2024-11-25T08:20:15,658 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732522787663 2024-11-25T08:20:15,715 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732522796282 2024-11-25T08:20:15,715 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-25T08:20:15,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741879_1055 (size=130) 2024-11-25T08:20:16,125 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42243/backupUT/backup_1732522796282/.backup.manifest 2024-11-25T08:20:16,127 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732522796282 finished: type=FULL,tablelist=ns2:test-17325227876631;test-1732522787663,targetRootDir=hdfs://localhost:42243/backupUT,startts=1732522802007,completets=1732522815713,bytescopied=0 2024-11-25T08:20:16,128 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-25T08:20:16,128 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:16,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732522806696_ns2_test-17325227876631" type: DISABLED 2024-11-25T08:20:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732522806696_ns2_test-17325227876631 2024-11-25T08:20:16,140 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732522806696_ns2_test-17325227876631 for backup backup_1732522796282 succeeded. 2024-11-25T08:20:16,140 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:16,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732522802513_default_test-1732522787663" type: DISABLED 2024-11-25T08:20:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732522802513_default_test-1732522787663 2024-11-25T08:20:16,145 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732522802513_default_test-1732522787663 for backup backup_1732522796282 succeeded. 2024-11-25T08:20:16,146 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-25T08:20:16,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-25T08:20:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-25T08:20:16,162 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-25T08:20:16,168 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-25T08:20:16,184 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732522796282 completed. 2024-11-25T08:20:16,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1732522796282', locateType=CURRENT is [region=backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=2] 2024-11-25T08:20:16,380 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1732522787663 2024-11-25T08:20:16,397 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1732522787663 2024-11-25T08:20:16,560 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1732522787663 to Mob enabled CF 2024-11-25T08:20:16,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17325227876631', row='row-t20', locateType=CURRENT is [region=ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=2] 2024-11-25T08:20:16,589 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17325227876631 2024-11-25T08:20:16,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:16,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] assignment.SplitTableRegionProcedure(210): test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. splitKey isn't explicitly specified, will try to find a best split key from RS 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:16,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b because midkey is the same as first or last row 2024-11-25T08:20:16,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27 because midkey is the same as first or last row 2024-11-25T08:20:16,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 because midkey is the same as first or last row 2024-11-25T08:20:16,601 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.HRegion(2902): Flushing 9c7bcb575c3e97a1ab0077b07fb801c5 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-25T08:20:16,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f/60394def41e549aca954d45be1e14ed2 is 59, key is row-test-1732522787663-1-10/f:q1/1732522816229/Put/seqid=0 2024-11-25T08:20:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741880_1056 (size=10796) 2024-11-25T08:20:17,027 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f/60394def41e549aca954d45be1e14ed2 2024-11-25T08:20:17,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5 is 63, key is row-test-1732522787663-3-100/mob:q1/1732522816545/Put/seqid=0 2024-11-25T08:20:17,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741881_1057 (size=5591) 2024-11-25T08:20:17,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:17,058 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/mobdir/data/default/test-1732522787663/6f9de7d24d3c4b766b3d2632af4ccf70/mob/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:17,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/mob/c746e2b059854d03a1f0eef704d4e9eb, store: [table=test-1732522787663 family=mob region=9c7bcb575c3e97a1ab0077b07fb801c5] 2024-11-25T08:20:17,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/mob/c746e2b059854d03a1f0eef704d4e9eb is 192, key is row-test-1732522787663-3-100/mob:q1/1732522816545/Put/seqid=0 2024-11-25T08:20:17,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741882_1058 (size=13760) 2024-11-25T08:20:17,465 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/mob/c746e2b059854d03a1f0eef704d4e9eb 2024-11-25T08:20:17,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f/60394def41e549aca954d45be1e14ed2 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/60394def41e549aca954d45be1e14ed2 2024-11-25T08:20:17,480 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/60394def41e549aca954d45be1e14ed2, entries=99, sequenceid=334, filesize=10.5 K 2024-11-25T08:20:17,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/mob/c746e2b059854d03a1f0eef704d4e9eb as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/c746e2b059854d03a1f0eef704d4e9eb 2024-11-25T08:20:17,488 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/c746e2b059854d03a1f0eef704d4e9eb, entries=111, sequenceid=334, filesize=13.4 K 2024-11-25T08:20:17,489 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 9c7bcb575c3e97a1ab0077b07fb801c5 in 888ms, sequenceid=334, compaction requested=false 2024-11-25T08:20:17,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.HRegion(2603): Flush status journal for 9c7bcb575c3e97a1ab0077b07fb801c5: 2024-11-25T08:20:17,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/60394def41e549aca954d45be1e14ed2 because midkey is the same as first or last row 2024-11-25T08:20:17,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27 because midkey is the same as first or last row 2024-11-25T08:20:17,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/c746e2b059854d03a1f0eef704d4e9eb because midkey is the same as first or last row 2024-11-25T08:20:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-25T08:20:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.CallRunner(138): callId: 302 service: MasterService methodName: SplitRegion size: 98 connection: 172.17.0.2:60972 deadline: 1732522876595, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-25T08:20:17,589 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:17,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1732522787663 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:17,590 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T08:20:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732522787663,, stopping at row=test-1732522787663 ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,604 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732522787663,, stopping at row=test-1732522787663 ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732522787663', row='', locateType=CURRENT is [region=test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=107] 2024-11-25T08:20:17,615 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1026 2024-11-25T08:20:17,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,659 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-25T08:20:17,664 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732522817615 2024-11-25T08:20:17,671 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732522796282} 2024-11-25T08:20:17,671 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-25T08:20:17,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:17,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:20:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522817716 (current time:1732522817716). 2024-11-25T08:20:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-25T08:20:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f3c14b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:17,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:17,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:17,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:17,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f28292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:17,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:17,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:17,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:17,720 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:17,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a050385, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:17,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:17,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:17,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:17,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:17,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:20:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-25T08:20:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T08:20:17,729 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:17,730 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:17,733 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:17,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741883_1059 (size=52) 2024-11-25T08:20:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T08:20:18,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T08:20:18,140 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:18,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909}] 2024-11-25T08:20:18,141 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:18,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-25T08:20:18,293 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:20:18,294 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 66dbbb90362d3aeb47c6625cd227a909 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-25T08:20:18,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/cea31546a83545cfb8ba5d035493ebb5 is 158, key is trslm:hdfs://localhost:42243/backupUT\x00ns2:test-17325227876631/meta:log-roll-map/1732522815700/Put/seqid=0 2024-11-25T08:20:18,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741884_1060 (size=5914) 2024-11-25T08:20:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T08:20:18,717 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/cea31546a83545cfb8ba5d035493ebb5 2024-11-25T08:20:18,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/08222de5beb242849698271cb3775eb4 is 433, key is session:backup_1732522796282/session:context/1732522816167/Put/seqid=0 2024-11-25T08:20:18,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741885_1061 (size=5535) 2024-11-25T08:20:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T08:20:19,134 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/08222de5beb242849698271cb3775eb4 2024-11-25T08:20:19,142 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/cea31546a83545cfb8ba5d035493ebb5 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5 2024-11-25T08:20:19,149 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5, entries=6, sequenceid=17, filesize=5.8 K 2024-11-25T08:20:19,150 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/08222de5beb242849698271cb3775eb4 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4 2024-11-25T08:20:19,158 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4, entries=2, sequenceid=17, filesize=5.4 K 2024-11-25T08:20:19,159 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 66dbbb90362d3aeb47c6625cd227a909 in 865ms, sequenceid=17, compaction requested=false 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 66dbbb90362d3aeb47c6625cd227a909: 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. for snapshot_backup_system completed. 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.' region-info for snapshot=snapshot_backup_system 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5] hfiles 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5 for snapshot=snapshot_backup_system 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4] hfiles 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b for snapshot=snapshot_backup_system 2024-11-25T08:20:19,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4 for snapshot=snapshot_backup_system 2024-11-25T08:20:19,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741886_1062 (size=173) 2024-11-25T08:20:19,566 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:20:19,566 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-25T08:20:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-25T08:20:19,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:19,567 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:19,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-25T08:20:19,570 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:19,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 in 1.4280 sec 2024-11-25T08:20:19,571 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:19,572 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:19,572 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-25T08:20:19,573 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-25T08:20:19,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741887_1063 (size=810) 2024-11-25T08:20:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T08:20:19,984 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:19,992 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:19,993 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_backup_system 2024-11-25T08:20:19,994 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:19,994 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-25T08:20:19,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2680 sec 2024-11-25T08:20:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-25T08:20:21,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-25T08:20:21,887 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732522817615 started at 1732522821887. 2024-11-25T08:20:21,889 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732522817615 has been started. 2024-11-25T08:20:21,894 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17325227876631, test-1732522787663] 2024-11-25T08:20:21,898 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732522788582for backupID backup_1732522817615 2024-11-25T08:20:21,898 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-25T08:20:21,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-25T08:20:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-25T08:20:21,899 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-25T08:20:21,899 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-25T08:20:21,899 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-25T08:20:21,899 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-25T08:20:21,900 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:21,900 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:21,935 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:21,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-25T08:20:21,936 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-25T08:20:21,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:20:21,936 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:21,936 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-25T08:20:21,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:21,937 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:21,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-25T08:20:21,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:21,937 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-25T08:20:21,938 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-25T08:20:21,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-25T08:20:21,938 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-25T08:20:21,938 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-25T08:20:21,938 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-25T08:20:21,938 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-25T08:20:21,938 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '5eb3d201e8c9,43089,1732522777179' joining acquired barrier for procedure (rolllog) in zk 2024-11-25T08:20:21,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:21,989 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:21,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:21,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:21,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:21,990 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-25T08:20:21,990 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-25T08:20:21,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:21,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:21,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:21,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:21,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:21,992 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '5eb3d201e8c9,43089,1732522777179' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-25T08:20:21,992 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@428147ec[Count = 0] remaining members to acquire global barrier 2024-11-25T08:20:21,992 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-25T08:20:21,992 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:22,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:22,036 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:22,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:22,036 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-25T08:20:22,036 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,036 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-25T08:20:22,036 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-25T08:20:22,036 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,036 INFO [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732522802286 highest: 1732522802286 on 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,037 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.meta:.meta(num 1732522802223) roll requested 2024-11-25T08:20:22,050 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522822037.meta, exclude list is [], retry=0 2024-11-25T08:20:22,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:22,055 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522802223.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522822037.meta 2024-11-25T08:20:22,055 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:22,055 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522802223.meta is not closed yet, will try archiving it next time 2024-11-25T08:20:22,055 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1:(num 1732522802253) roll requested 2024-11-25T08:20:22,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741857_1033 (size=93) 2024-11-25T08:20:22,058 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522802223.meta to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522802223.meta 2024-11-25T08:20:22,070 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522822056, exclude list is [], retry=0 2024-11-25T08:20:22,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:22,073 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522822056 2024-11-25T08:20:22,073 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:22,074 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 is not closed yet, will try archiving it next time 2024-11-25T08:20:22,074 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0:(num 1732522802286) roll requested 2024-11-25T08:20:22,074 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522788582 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522788582 2024-11-25T08:20:22,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741858_1034 (size=30490) 2024-11-25T08:20:22,087 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522822074, exclude list is [], retry=0 2024-11-25T08:20:22,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:22,091 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522822074 2024-11-25T08:20:22,092 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:22,092 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 is not closed yet, will try archiving it next time 2024-11-25T08:20:22,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741859_1035 (size=1198) 2024-11-25T08:20:22,097 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-25T08:20:22,097 INFO [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732522822074 on 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:22,112 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:22,118 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-25T08:20:22,118 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '5eb3d201e8c9,43089,1732522777179' in zk 2024-11-25T08:20:22,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,159 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-25T08:20:22,159 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-25T08:20:22,159 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-25T08:20:22,159 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,159 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:22,159 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:22,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:22,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:22,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:22,161 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:22,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:22,162 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,163 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '5eb3d201e8c9,43089,1732522777179': 2024-11-25T08:20:22,163 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-25T08:20:22,163 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-25T08:20:22,163 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '5eb3d201e8c9,43089,1732522777179' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-25T08:20:22,164 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-25T08:20:22,164 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-25T08:20:22,164 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-25T08:20:22,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:22,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:22,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-25T08:20:22,198 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:22,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:22,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:22,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:22,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:22,199 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-25T08:20:22,199 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:20:22,199 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:22,199 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,199 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:22,200 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:22,200 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:22,200 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:22,201 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,201 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,201 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:22,202 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:22,202 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:22,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-25T08:20:22,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:22,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:22,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:22,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:22,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:22,228 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-25T08:20:22,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:20:22,228 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-25T08:20:22,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-25T08:20:22,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-25T08:20:22,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-25T08:20:22,229 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-25T08:20:22,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:20:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-25T08:20:22,240 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {5eb3d201e8c9:43089=1732522788582} newestTimestamps: {5eb3d201e8c9:43089=1732522802286} 2024-11-25T08:20:22,242 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522782479.meta 2024-11-25T08:20:22,243 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522822037.meta 2024-11-25T08:20:22,243 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 2024-11-25T08:20:22,243 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522822074 2024-11-25T08:20:22,243 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 2024-11-25T08:20:22,243 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522822056 2024-11-25T08:20:22,244 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522802223.meta 2024-11-25T08:20:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741891_1067 (size=364) 2024-11-25T08:20:22,656 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17325227876631 target: hdfs://localhost:42243/backupUT/backup_1732522817615/ns2/test-17325227876631 descriptor: 'ns2:test-17325227876631', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:20:22,656 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-25T08:20:22,659 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17325227876631 2024-11-25T08:20:22,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741892_1068 (size=50) 2024-11-25T08:20:22,899 DEBUG [master/5eb3d201e8c9:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732522787663 2024-11-25T08:20:22,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741893_1069 (size=950) 2024-11-25T08:20:23,065 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17325227876631 2024-11-25T08:20:23,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741894_1070 (size=950) 2024-11-25T08:20:23,313 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/.tabledesc/.tableinfo.0000000002.908 2024-11-25T08:20:23,313 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/.tabledesc/.tableinfo.0000000003.950 2024-11-25T08:20:23,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732522787663 in 417 msec 2024-11-25T08:20:23,478 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732522787663 target: hdfs://localhost:42243/backupUT/backup_1732522817615/default/test-1732522787663 descriptor: 'test-1732522787663', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:20:23,478 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-25T08:20:23,481 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732522787663 2024-11-25T08:20:23,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741895_1071 (size=53) 2024-11-25T08:20:23,888 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732522787663 2024-11-25T08:20:23,898 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:42243/backupUT/.tmp/backup_1732522817615 from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286;hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 2024-11-25T08:20:23,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c1e2ecf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:23,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:23,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:23,901 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:23,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:23,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:23,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b987d77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:23,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:23,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:23,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:23,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:23,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778c5156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:23,904 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T08:20:23,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5eb3d201e8c9,44851,1732522775045 2024-11-25T08:20:23,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f5d4ac3 2024-11-25T08:20:23,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T08:20:23,905 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T08:20:23,911 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-25T08:20:23,911 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17325227876631 2024-11-25T08:20:23,911 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17325227876631,, stopping at row=ns2:test-17325227876631 ,, for max=2147483647 with caching=100 2024-11-25T08:20:23,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:23,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:23,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:23,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:23,916 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17325227876631: ns2:test-17325227876631; 2024-11-25T08:20:23,917 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732522787663 2024-11-25T08:20:23,917 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732522787663,, stopping at row=test-1732522787663 ,, for max=2147483647 with caching=100 2024-11-25T08:20:23,919 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732522787663: test-1732522787663; 2024-11-25T08:20:23,919 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-25T08:20:23,919 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/hbase-staging/partitions_2c5439f1-d938-4228-971e-389ff3100692 2024-11-25T08:20:23,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741896_1072 (size=192) 2024-11-25T08:20:24,346 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:24,346 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:24,346 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,342 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-17282103868092662918.jar 2024-11-25T08:20:25,342 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,342 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-2846568227751040810.jar 2024-11-25T08:20:25,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,409 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,410 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:20:25,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:20:25,411 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:20:25,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:20:25,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:20:25,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:20:25,412 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:20:25,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:20:25,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:20:25,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:20:25,413 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:20:25,415 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,481 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-3406251012626418619.jar 2024-11-25T08:20:25,481 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-3406251012626418619.jar 2024-11-25T08:20:25,482 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:25,482 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-3406251012626418619.jar 2024-11-25T08:20:25,482 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-3406251012626418619.jar 2024-11-25T08:20:25,483 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:25,483 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17325227876631,default:test-1732522787663 2024-11-25T08:20:25,483 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T08:20:25,483 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:20:25,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:25,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:25,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:25,484 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:20:26,473 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-12549094293682495239.jar 2024-11-25T08:20:26,487 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:20:26,502 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:20:26,507 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 for WAL files 2024-11-25T08:20:26,509 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 2024-11-25T08:20:26,509 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 for WAL files 2024-11-25T08:20:26,510 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 2024-11-25T08:20:26,625 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local1796566499_0003_c8c35c50-1630-4544-8864-a961f00e5514/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-25T08:20:26,625 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local1796566499_0003_c8c35c50-1630-4544-8864-a961f00e5514/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-25T08:20:26,730 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 for hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 (-9223372036854775808:9223372036854775807) length:30490 2024-11-25T08:20:26,755 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-25T08:20:26,756 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-25T08:20:26,959 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 for hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 (-9223372036854775808:9223372036854775807) length:1198 2024-11-25T08:20:26,963 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-25T08:20:26,963 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-25T08:20:27,062 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d258173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,063 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:27,063 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:27,064 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:27,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:27,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:27,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74098e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:27,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:27,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:27,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2765a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,067 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:27,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:27,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:27,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:27,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17325227876631', row='row-t20', locateType=CURRENT is [region=ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=2] 2024-11-25T08:20:27,072 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-25T08:20:27,072 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:20:27,072 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,072 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:27,073 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-25T08:20:27,073 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5eb3d201e8c9 2024-11-25T08:20:27,079 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/backupUT/.tmp/backup_1732522817615/_temporary/0/_temporary/attempt_local1796566499_0003_r_000000_0/ns2/test-17325227876631/f/0192a10f99f6494d81eb48a142abaa52 is 38, key is row-t20/f:q1/1732522816576/Put/seqid=0 2024-11-25T08:20:27,082 WARN [Thread-944 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5eb3d201e8c9:43089] Specified favored nodes: [5eb3d201e8c9:43089] 2024-11-25T08:20:27,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741897_1073 (size=5275) 2024-11-25T08:20:27,515 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9bc863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,515 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:27,515 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:27,517 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:27,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:27,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:27,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55544811, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:27,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:27,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,518 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:27,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@572a8238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,519 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:27,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:27,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:27,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:27,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732522787663', row='row-test-1732522787663-1-0', locateType=CURRENT is [region=test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=107] 2024-11-25T08:20:27,523 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-25T08:20:27,523 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:20:27,523 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,523 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:27,524 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732522787663-1-0] 2024-11-25T08:20:27,524 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5eb3d201e8c9 2024-11-25T08:20:27,537 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48972212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,537 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:27,537 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:27,538 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:27,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:27,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:27,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf338cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:27,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:27,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,540 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:27,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd03b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:27,541 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:27,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:27,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:27,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:27,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732522787663', row='row-test-1732522787663-3-0', locateType=CURRENT is [region=test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=107] 2024-11-25T08:20:27,545 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-25T08:20:27,545 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:20:27,545 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,545 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:27,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:27,545 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732522787663-3-0] 2024-11-25T08:20:27,545 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 5eb3d201e8c9 2024-11-25T08:20:27,557 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/backupUT/.tmp/backup_1732522817615/_temporary/0/_temporary/attempt_local1796566499_0003_r_000001_0/default/test-1732522787663/f/a0eea274209a472589c672f7aa300640 is 59, key is row-test-1732522787663-1-10/f:q1/1732522816229/Put/seqid=0 2024-11-25T08:20:27,559 WARN [Thread-956 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5eb3d201e8c9:43089] Specified favored nodes: [5eb3d201e8c9:43089] 2024-11-25T08:20:27,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741898_1074 (size=10770) 2024-11-25T08:20:27,962 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/backupUT/.tmp/backup_1732522817615/_temporary/0/_temporary/attempt_local1796566499_0003_r_000001_0/default/test-1732522787663/mob/05fbe8522d24427dbeb86dba1856761e is 63, key is row-test-1732522787663-3-100/mob:q1/1732522816545/Put/seqid=0 2024-11-25T08:20:27,964 WARN [Thread-958 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [5eb3d201e8c9:43089] Specified favored nodes: [5eb3d201e8c9:43089] 2024-11-25T08:20:27,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741899_1075 (size=11811) 2024-11-25T08:20:28,669 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:42243/backupUT 2024-11-25T08:20:28,669 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732522817615 2024-11-25T08:20:28,669 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-25T08:20:28,689 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:42243/backupUT/.tmp/backup_1732522817615, hdfs://localhost:42243/backupUT] 2024-11-25T08:20:28,765 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:20:28,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741900_1076 (size=11811) 2024-11-25T08:20:29,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741901_1077 (size=10770) 2024-11-25T08:20:29,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741902_1078 (size=5275) 2024-11-25T08:20:30,397 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-25T08:20:30,400 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732522817615 2024-11-25T08:20:30,400 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-11-25T08:20:30,401 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local193016252_0004 completed: true true 2024-11-25T08:20:30,401 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=2493802 FILE: Number of bytes written=4873900 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231467 HDFS: Number of bytes written=2433185 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=250 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-11-25T08:20:30,401 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:42243/backupUT/.tmp/backup_1732522817615 to hdfs://localhost:42243/backupUT finished. 2024-11-25T08:20:30,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:ns2:test-17325227876631:', locateType=CURRENT is [region=backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364., hostname=5eb3d201e8c9,43089,1732522777179, seqNum=2] 2024-11-25T08:20:30,421 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732522817615 2024-11-25T08:20:30,425 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732522796282} 2024-11-25T08:20:30,425 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-25T08:20:30,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741903_1079 (size=384) 2024-11-25T08:20:30,831 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42243/backupUT/backup_1732522817615/.backup.manifest 2024-11-25T08:20:30,831 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732522817615 finished: type=INCREMENTAL,tablelist=ns2:test-17325227876631;test-1732522787663,targetRootDir=hdfs://localhost:42243/backupUT,startts=1732522821887,completets=1732522830421,bytescopied=0 2024-11-25T08:20:30,832 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-25T08:20:30,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-25T08:20:30,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-25T08:20:30,841 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-25T08:20:30,844 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-25T08:20:30,849 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732522817615 completed. 2024-11-25T08:20:30,849 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-25T08:20:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:30,868 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42243/backupUT/backup_1732522817615/.backup.manifest 2024-11-25T08:20:30,871 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732522817615/.backup.manifest 2024-11-25T08:20:30,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732522787663 from 'test-1732522787663', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732522787663', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:20:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732522787663 2024-11-25T08:20:30,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741904_1080 (size=908) 2024-11-25T08:20:30,888 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/.tabledesc/.tableinfo.0000000003.950 2024-11-25T08:20:30,888 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/.tabledesc/.tableinfo.0000000004.908 2024-11-25T08:20:30,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732522787663}] 2024-11-25T08:20:30,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE}] 2024-11-25T08:20:30,895 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE 2024-11-25T08:20:30,897 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=CLOSING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:30,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE because future has completed 2024-11-25T08:20:30,899 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T08:20:30,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:20:31,052 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,052 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T08:20:31,053 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 9c7bcb575c3e97a1ab0077b07fb801c5, disabling compactions & flushes 2024-11-25T08:20:31,053 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:31,053 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:31,053 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. after waiting 0 ms 2024-11-25T08:20:31,053 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:31,059 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-25T08:20:31,060 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:20:31,060 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:31,060 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 9c7bcb575c3e97a1ab0077b07fb801c5: Waiting for close lock at 1732522831052Running coprocessor pre-close hooks at 1732522831052Disabling compacts and flushes for region at 1732522831052Disabling writes for close at 1732522831053 (+1 ms)Writing region close event to WAL at 1732522831055 (+2 ms)Running coprocessor post-close hooks at 1732522831060 (+5 ms)Closed at 1732522831060 2024-11-25T08:20:31,060 WARN [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 9c7bcb575c3e97a1ab0077b07fb801c5 to self. 2024-11-25T08:20:31,063 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,063 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=CLOSED 2024-11-25T08:20:31,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:20:31,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-25T08:20:31,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 in 167 msec 2024-11-25T08:20:31,069 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE; state=CLOSED, location=5eb3d201e8c9,43089,1732522777179; forceNewPlan=false, retain=true 2024-11-25T08:20:31,220 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:31,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE because future has completed 2024-11-25T08:20:31,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:20:31,379 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:31,379 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 9c7bcb575c3e97a1ab0077b07fb801c5, NAME => 'test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:20:31,380 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:20:31,380 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732522787663 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,380 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:20:31,380 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,380 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,382 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,383 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c7bcb575c3e97a1ab0077b07fb801c5 columnFamilyName f 2024-11-25T08:20:31,383 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:31,390 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/60394def41e549aca954d45be1e14ed2 2024-11-25T08:20:31,395 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b 2024-11-25T08:20:31,395 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(327): Store=9c7bcb575c3e97a1ab0077b07fb801c5/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:20:31,395 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,397 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c7bcb575c3e97a1ab0077b07fb801c5 columnFamilyName f2 2024-11-25T08:20:31,397 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:31,398 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(327): Store=9c7bcb575c3e97a1ab0077b07fb801c5/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:20:31,398 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,399 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9c7bcb575c3e97a1ab0077b07fb801c5 columnFamilyName mob 2024-11-25T08:20:31,399 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:31,407 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 2024-11-25T08:20:31,412 DEBUG [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/c746e2b059854d03a1f0eef704d4e9eb 2024-11-25T08:20:31,412 INFO [StoreOpener-9c7bcb575c3e97a1ab0077b07fb801c5-1 {}] regionserver.HStore(327): Store=9c7bcb575c3e97a1ab0077b07fb801c5/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:20:31,413 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,413 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,414 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,415 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,415 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,416 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732522787663 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T08:20:31,418 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,419 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 9c7bcb575c3e97a1ab0077b07fb801c5; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10220238560, jitterRate=-0.04816611111164093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T08:20:31,419 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:31,421 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 9c7bcb575c3e97a1ab0077b07fb801c5: Running coprocessor pre-open hook at 1732522831380Writing region info on filesystem at 1732522831380Initializing all the Stores at 1732522831381 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522831381Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522831382 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522831382Cleaning up temporary data from old regions at 1732522831415 (+33 ms)Running coprocessor post-open hooks at 1732522831419 (+4 ms)Region opened successfully at 1732522831421 (+2 ms) 2024-11-25T08:20:31,422 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., pid=46, masterSystemTime=1732522831374 2024-11-25T08:20:31,425 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:31,425 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:31,425 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=9c7bcb575c3e97a1ab0077b07fb801c5, regionState=OPEN, openSeqNum=338, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:31,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:20:31,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-25T08:20:31,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5, server=5eb3d201e8c9,43089,1732522777179 in 205 msec 2024-11-25T08:20:31,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-25T08:20:31,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732522787663, region=9c7bcb575c3e97a1ab0077b07fb801c5, REOPEN/MOVE in 536 msec 2024-11-25T08:20:31,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-25T08:20:31,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732522787663 in 542 msec 2024-11-25T08:20:31,437 DEBUG [PEWorker-5 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1732522787663 2024-11-25T08:20:31,444 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f3/d4be02add9f447c3b5cf47114241bc27 2024-11-25T08:20:31,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732522787663 in 572 msec 2024-11-25T08:20:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T08:20:31,448 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732522787663 completed 2024-11-25T08:20:31,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,484 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-25T08:20:31,488 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732522831449 2024-11-25T08:20:31,490 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1732522817615} 2024-11-25T08:20:31,490 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732522796282} 2024-11-25T08:20:31,490 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-25T08:20:31,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,537 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1732522787663 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1732522787663 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T08:20:31,541 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732522831449,startts=0,failedts=1732522831537,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1732522787663 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-25T08:20:31,541 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-25T08:20:31,543 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-25T08:20:31,543 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-25T08:20:31,545 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-25T08:20:31,545 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-25T08:20:31,548 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-25T08:20:31,550 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732522831449 failed. 2024-11-25T08:20:31,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:31,570 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1732522787663, ns2:test-17325227876631, ns3:test-17325227876632, ns4:test-17325227876633] 2024-11-25T08:20:31,571 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-25T08:20:31,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:20:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522831576 (current time:1732522831576). 2024-11-25T08:20:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-25T08:20:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e7c6639, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:31,577 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:31,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:31,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:31,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c452ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:31,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:31,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:31,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:31,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:31,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2495feea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:31,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:31,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:31,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:31,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:31,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:20:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-25T08:20:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-25T08:20:31,586 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:31,587 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:31,590 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:31,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741905_1081 (size=52) 2024-11-25T08:20:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-25T08:20:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-25T08:20:31,997 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:31,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909}] 2024-11-25T08:20:31,998 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:32,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-11-25T08:20:32,150 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:20:32,150 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing 66dbbb90362d3aeb47c6625cd227a909 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-25T08:20:32,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/a618dcdf0f4b4c91889239c79c4576bc is 158, key is trslm:hdfs://localhost:42243/backupUT\x00ns2:test-17325227876631/meta:log-roll-map/1732522830408/Put/seqid=0 2024-11-25T08:20:32,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741906_1082 (size=5745) 2024-11-25T08:20:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-25T08:20:32,560 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/a618dcdf0f4b4c91889239c79c4576bc 2024-11-25T08:20:32,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/d7c07d95a1c34d869ecba7f8f3cd5b06 is 1245, key is session:backup_1732522831449/session:context/1732522831547/Put/seqid=0 2024-11-25T08:20:32,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741907_1083 (size=6682) 2024-11-25T08:20:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-25T08:20:32,865 DEBUG [master/5eb3d201e8c9:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732522787663 2024-11-25T08:20:32,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741908_1084 (size=950) 2024-11-25T08:20:32,976 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/d7c07d95a1c34d869ecba7f8f3cd5b06 2024-11-25T08:20:32,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/a618dcdf0f4b4c91889239c79c4576bc as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc 2024-11-25T08:20:32,989 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc, entries=4, sequenceid=31, filesize=5.6 K 2024-11-25T08:20:32,990 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/d7c07d95a1c34d869ecba7f8f3cd5b06 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06 2024-11-25T08:20:32,997 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06, entries=3, sequenceid=31, filesize=6.5 K 2024-11-25T08:20:32,998 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 66dbbb90362d3aeb47c6625cd227a909 in 848ms, sequenceid=31, compaction requested=true 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for 66dbbb90362d3aeb47c6625cd227a909: 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. for snapshot_backup_system completed. 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.' region-info for snapshot=snapshot_backup_system 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc] hfiles 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5 for snapshot=snapshot_backup_system 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc for snapshot=snapshot_backup_system 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06] hfiles 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b for snapshot=snapshot_backup_system 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4 for snapshot=snapshot_backup_system 2024-11-25T08:20:32,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06 for snapshot=snapshot_backup_system 2024-11-25T08:20:33,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741909_1085 (size=251) 2024-11-25T08:20:33,277 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/.tabledesc/.tableinfo.0000000004.908 2024-11-25T08:20:33,277 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/.tabledesc/.tableinfo.0000000005.950 2024-11-25T08:20:33,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732522787663 in 413 msec 2024-11-25T08:20:33,404 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:20:33,404 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-25T08:20:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-11-25T08:20:33,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:33,405 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:20:33,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-25T08:20:33,408 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:33,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 in 1.4090 sec 2024-11-25T08:20:33,408 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:33,409 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:33,409 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-25T08:20:33,410 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-25T08:20:33,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741910_1086 (size=888) 2024-11-25T08:20:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-25T08:20:33,821 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:33,829 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:33,830 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_backup_system 2024-11-25T08:20:33,831 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:33,831 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-25T08:20:33,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2470 sec 2024-11-25T08:20:34,092 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T08:20:35,153 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 74322db2b59a5db2ea28349c44aa1b80, had cached 0 bytes from a total of 8413 2024-11-25T08:20:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-25T08:20:35,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-25T08:20:35,747 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732522831552 started at 1732522835746. 2024-11-25T08:20:35,749 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732522831552 has been started. 2024-11-25T08:20:35,753 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-25T08:20:35,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-25T08:20:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-25T08:20:35,754 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-25T08:20:35,754 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-25T08:20:35,754 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-25T08:20:35,754 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-25T08:20:35,755 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,755 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:35,764 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-25T08:20:35,764 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-25T08:20:35,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:20:35,764 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,764 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-25T08:20:35,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:35,765 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-25T08:20:35,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:35,766 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-25T08:20:35,766 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-25T08:20:35,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-25T08:20:35,766 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-25T08:20:35,766 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-25T08:20:35,766 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-25T08:20:35,766 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-25T08:20:35,766 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '5eb3d201e8c9,43089,1732522777179' joining acquired barrier for procedure (rolllog) in zk 2024-11-25T08:20:35,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,779 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:35,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:35,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:35,779 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-25T08:20:35,779 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-25T08:20:35,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:35,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:35,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:35,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:35,782 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '5eb3d201e8c9,43089,1732522777179' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-25T08:20:35,782 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6da18da5[Count = 0] remaining members to acquire global barrier 2024-11-25T08:20:35,782 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-25T08:20:35,782 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:35,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:35,794 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:35,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:35,795 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-25T08:20:35,795 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,795 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-25T08:20:35,795 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-25T08:20:35,795 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,795 INFO [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732522822074 highest: 1732522822074 on 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,795 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.meta:.meta(num 1732522822037) roll requested 2024-11-25T08:20:35,809 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522835795.meta, exclude list is [], retry=0 2024-11-25T08:20:35,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:35,814 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522822037.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522835795.meta 2024-11-25T08:20:35,814 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:35,814 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522822037.meta is not closed yet, will try archiving it next time 2024-11-25T08:20:35,814 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1:(num 1732522822056) roll requested 2024-11-25T08:20:35,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741888_1064 (size=1903) 2024-11-25T08:20:35,816 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522822037.meta not finished, retry = 0 2024-11-25T08:20:35,828 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522835814, exclude list is [], retry=0 2024-11-25T08:20:35,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:35,831 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522822056 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522835814 2024-11-25T08:20:35,831 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:35,831 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522822056 is not closed yet, will try archiving it next time 2024-11-25T08:20:35,832 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0:(num 1732522822074) roll requested 2024-11-25T08:20:35,832 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522802253 2024-11-25T08:20:35,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741889_1065 (size=4927) 2024-11-25T08:20:35,845 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832, exclude list is [], retry=0 2024-11-25T08:20:35,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:20:35,849 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522822074 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 2024-11-25T08:20:35,849 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:20:35,849 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522822074 is not closed yet, will try archiving it next time 2024-11-25T08:20:35,850 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522822074 not finished, retry = 0 2024-11-25T08:20:35,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741890_1066 (size=93) 2024-11-25T08:20:35,855 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-25T08:20:35,855 INFO [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732522835832 on 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:20:35,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:20:35,871 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-25T08:20:35,871 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '5eb3d201e8c9,43089,1732522777179' in zk 2024-11-25T08:20:35,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,882 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-25T08:20:35,882 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-25T08:20:35,882 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-25T08:20:35,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:35,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:35,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:35,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:35,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:35,883 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:35,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:35,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '5eb3d201e8c9,43089,1732522777179': 2024-11-25T08:20:35,885 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-25T08:20:35,885 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '5eb3d201e8c9,43089,1732522777179' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-25T08:20:35,885 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-25T08:20:35,885 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-25T08:20:35,885 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-25T08:20:35,885 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-25T08:20:35,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-25T08:20:35,895 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:20:35,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:20:35,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:20:35,895 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-25T08:20:35,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:20:35,895 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:35,896 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,896 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:20:35,896 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:35,897 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,897 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:20:35,897 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,897 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:20:35,898 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:20:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-25T08:20:35,910 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-25T08:20:35,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:20:35,910 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-25T08:20:35,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-25T08:20:35,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-25T08:20:35,910 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-25T08:20:35,910 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:20:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-25T08:20:35,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-25T08:20:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522835918 (current time:1732522835918). 2024-11-25T08:20:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732522835918_backup_system_bulk VERSION not specified, setting to 2 2024-11-25T08:20:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d28044e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:35,920 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:35,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:35,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:35,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba575e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:35,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:35,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:35,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:35,922 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:35,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7528b83b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:35,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:35,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:35,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:35,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:35,925 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-25T08:20:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-25T08:20:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T08:20:35,928 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:35,929 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:35,931 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:35,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741914_1090 (size=76) 2024-11-25T08:20:35,951 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522822074 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522822074 2024-11-25T08:20:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T08:20:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T08:20:36,338 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:36,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure daf94d1506be6ea0f96237e4c58ae364}] 2024-11-25T08:20:36,339 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:20:36,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-25T08:20:36,490 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:20:36,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for daf94d1506be6ea0f96237e4c58ae364: 2024-11-25T08:20:36,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. for snapshot_1732522835918_backup_system_bulk completed. 2024-11-25T08:20:36,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364.' region-info for snapshot=snapshot_1732522835918_backup_system_bulk 2024-11-25T08:20:36,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:36,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-25T08:20:36,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-25T08:20:36,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741915_1091 (size=61) 2024-11-25T08:20:36,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T08:20:36,585 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-25T08:20:36,897 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:20:36,897 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-25T08:20:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-25T08:20:36,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732522835918_backup_system_bulk on region daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:20:36,898 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:20:36,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-25T08:20:36,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure daf94d1506be6ea0f96237e4c58ae364 in 561 msec 2024-11-25T08:20:36,901 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:36,902 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:36,902 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:36,902 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732522835918_backup_system_bulk 2024-11-25T08:20:36,903 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522835918_backup_system_bulk 2024-11-25T08:20:36,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741916_1092 (size=702) 2024-11-25T08:20:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T08:20:37,313 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:37,318 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:37,318 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522835918_backup_system_bulk to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522835918_backup_system_bulk 2024-11-25T08:20:37,320 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:37,320 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-25T08:20:37,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732522835918_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.3940 sec 2024-11-25T08:20:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T08:20:38,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-25T08:20:38,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } 2024-11-25T08:20:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522838077 (current time:1732522838077). 2024-11-25T08:20:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732522838076_ns4_test-17325227876633 VERSION not specified, setting to 2 2024-11-25T08:20:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ce7db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:38,078 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:38,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:38,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:38,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1954a21c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:38,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:38,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:38,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:38,080 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:38,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b502c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:38,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:38,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:38,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:38,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:38,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } 2024-11-25T08:20:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-25T08:20:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T08:20:38,087 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:38,088 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:38,090 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:38,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741917_1093 (size=86) 2024-11-25T08:20:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T08:20:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T08:20:38,496 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:38,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44a4d8f1b8ed294e1ddee3eb2e0fe2e9}] 2024-11-25T08:20:38,497 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:20:38,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-25T08:20:38,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:20:38,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9: 2024-11-25T08:20:38,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. for snapshot_1732522838076_ns4_test-17325227876633 completed. 2024-11-25T08:20:38,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9.' region-info for snapshot=snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:20:38,649 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:38,650 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-25T08:20:38,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741918_1094 (size=52) 2024-11-25T08:20:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T08:20:39,056 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:20:39,056 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-25T08:20:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-25T08:20:39,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732522838076_ns4_test-17325227876633 on region 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:20:39,057 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:20:39,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-25T08:20:39,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 in 562 msec 2024-11-25T08:20:39,059 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:39,060 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:39,060 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:39,060 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:20:39,061 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:20:39,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741919_1095 (size=443) 2024-11-25T08:20:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T08:20:39,471 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:39,477 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:39,477 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522838076_ns4_test-17325227876633 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:20:39,479 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:39,479 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-25T08:20:39,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732522838076_ns4_test-17325227876633 table=ns4:test-17325227876633 type=FLUSH ttl=0 } in 1.3940 sec 2024-11-25T08:20:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T08:20:40,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17325227876633 completed 2024-11-25T08:20:40,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } 2024-11-25T08:20:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522840246 (current time:1732522840246). 2024-11-25T08:20:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732522840246_ns2_test-17325227876631 VERSION not specified, setting to 2 2024-11-25T08:20:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3503f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:40,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:40,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:40,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:40,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@155a3b6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:40,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:40,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:40,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:40,250 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:40,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d945ab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:40,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:40,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:40,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:40,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:40,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:40,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } 2024-11-25T08:20:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-25T08:20:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T08:20:40,256 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:40,257 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:40,260 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:40,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741920_1096 (size=86) 2024-11-25T08:20:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T08:20:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T08:20:40,666 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:40,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 74322db2b59a5db2ea28349c44aa1b80}] 2024-11-25T08:20:40,667 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:20:40,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-25T08:20:40,819 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:20:40,819 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 74322db2b59a5db2ea28349c44aa1b80 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-25T08:20:40,824 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/.tmp/f/a071fb1553b44115828b8af4e4cad1df is 38, key is row-t20/f:q1/1732522816576/Put/seqid=0 2024-11-25T08:20:40,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741921_1097 (size=5134) 2024-11-25T08:20:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T08:20:41,228 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/.tmp/f/a071fb1553b44115828b8af4e4cad1df 2024-11-25T08:20:41,235 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/.tmp/f/a071fb1553b44115828b8af4e4cad1df as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/a071fb1553b44115828b8af4e4cad1df 2024-11-25T08:20:41,241 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/a071fb1553b44115828b8af4e4cad1df, entries=5, sequenceid=111, filesize=5.0 K 2024-11-25T08:20:41,241 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 74322db2b59a5db2ea28349c44aa1b80 in 422ms, sequenceid=111, compaction requested=false 2024-11-25T08:20:41,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 74322db2b59a5db2ea28349c44aa1b80: 2024-11-25T08:20:41,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. for snapshot_1732522840246_ns2_test-17325227876631 completed. 2024-11-25T08:20:41,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80.' region-info for snapshot=snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:41,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:41,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/a071fb1553b44115828b8af4e4cad1df] hfiles 2024-11-25T08:20:41,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876 for snapshot=snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:41,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/a071fb1553b44115828b8af4e4cad1df for snapshot=snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:41,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741922_1098 (size=130) 2024-11-25T08:20:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T08:20:41,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:20:41,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-25T08:20:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-25T08:20:41,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732522840246_ns2_test-17325227876631 on region 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:20:41,649 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:20:41,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-25T08:20:41,671 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:41,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 74322db2b59a5db2ea28349c44aa1b80 in 1.0030 sec 2024-11-25T08:20:41,671 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:41,672 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:41,672 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:41,673 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:41,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741923_1099 (size=496) 2024-11-25T08:20:42,083 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:42,090 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:42,090 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522840246_ns2_test-17325227876631 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:42,092 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:42,092 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-25T08:20:42,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732522840246_ns2_test-17325227876631 table=ns2:test-17325227876631 type=FLUSH ttl=0 } in 1.8370 sec 2024-11-25T08:20:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-25T08:20:42,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17325227876631 completed 2024-11-25T08:20:42,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } 2024-11-25T08:20:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522842407 (current time:1732522842407). 2024-11-25T08:20:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732522842406_default_test-1732522787663 VERSION not specified, setting to 2 2024-11-25T08:20:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ee94ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:42,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:42,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:42,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:42,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6dc2d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:42,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:42,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:42,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:42,410 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:42,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff61486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:42,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:42,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:42,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:42,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:42,414 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } 2024-11-25T08:20:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-25T08:20:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T08:20:42,417 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:42,418 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:42,420 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:42,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741924_1100 (size=84) 2024-11-25T08:20:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T08:20:42,598 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 66dbbb90362d3aeb47c6625cd227a909, had cached 0 bytes from a total of 28925 2024-11-25T08:20:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T08:20:42,826 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:42,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5}] 2024-11-25T08:20:42,827 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:42,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-25T08:20:42,979 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 9c7bcb575c3e97a1ab0077b07fb801c5: 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. for snapshot_1732522842406_default_test-1732522787663 completed. 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5.' region-info for snapshot=snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/60394def41e549aca954d45be1e14ed2] hfiles 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b for snapshot=snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/60394def41e549aca954d45be1e14ed2 for snapshot=snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/c746e2b059854d03a1f0eef704d4e9eb] hfiles 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 for snapshot=snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:42,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/c746e2b059854d03a1f0eef704d4e9eb for snapshot=snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:42,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741925_1101 (size=224) 2024-11-25T08:20:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T08:20:43,187 INFO [master/5eb3d201e8c9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T08:20:43,187 INFO [master/5eb3d201e8c9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T08:20:43,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:20:43,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-25T08:20:43,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-25T08:20:43,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732522842406_default_test-1732522787663 on region 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:43,388 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:43,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-25T08:20:43,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9c7bcb575c3e97a1ab0077b07fb801c5 in 562 msec 2024-11-25T08:20:43,391 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:43,392 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:43,393 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-25T08:20:43,393 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-25T08:20:43,393 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:43,394 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-25T08:20:43,394 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:43,394 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-25T08:20:43,394 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:20:43,395 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/mobdir/data/default/test-1732522787663/6f9de7d24d3c4b766b3d2632af4ccf70/mob/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5] hfiles 2024-11-25T08:20:43,395 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/mobdir/data/default/test-1732522787663/6f9de7d24d3c4b766b3d2632af4ccf70/mob/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:43,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741926_1102 (size=168) 2024-11-25T08:20:43,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T08:20:43,802 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:43,802 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:43,803 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:43,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741927_1103 (size=1347) 2024-11-25T08:20:44,214 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:44,223 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:44,223 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522842406_default_test-1732522787663 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:44,225 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:44,225 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-25T08:20:44,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732522842406_default_test-1732522787663 table=test-1732522787663 type=FLUSH ttl=0 } in 1.8100 sec 2024-11-25T08:20:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T08:20:44,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732522787663 completed 2024-11-25T08:20:44,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } 2024-11-25T08:20:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522844576 (current time:1732522844576). 2024-11-25T08:20:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:20:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732522844576_ns3_test-17325227876632 VERSION not specified, setting to 2 2024-11-25T08:20:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4158b7cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:20:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:20:44,578 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:20:44,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:20:44,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:20:44,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d23ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:44,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:20:44,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:20:44,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:44,580 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:20:44,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68186e13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:20:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:20:44,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:20:44,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:20:44,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:20:44,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:20:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:20:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:20:44,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:20:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:20:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } 2024-11-25T08:20:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-25T08:20:44,586 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:20:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T08:20:44,587 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:20:44,589 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:20:44,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741928_1104 (size=86) 2024-11-25T08:20:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T08:20:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T08:20:44,996 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:20:44,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 53385123a893b8dd1a5dbd90b0513715}] 2024-11-25T08:20:44,997 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:20:45,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-25T08:20:45,149 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:20:45,149 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 53385123a893b8dd1a5dbd90b0513715: 2024-11-25T08:20:45,149 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. for snapshot_1732522844576_ns3_test-17325227876632 completed. 2024-11-25T08:20:45,149 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715.' region-info for snapshot=snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:20:45,149 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:20:45,149 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-25T08:20:45,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741929_1105 (size=52) 2024-11-25T08:20:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T08:20:45,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:20:45,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-25T08:20:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-25T08:20:45,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732522844576_ns3_test-17325227876632 on region 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:20:45,557 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:20:45,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-25T08:20:45,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 53385123a893b8dd1a5dbd90b0513715 in 562 msec 2024-11-25T08:20:45,559 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:20:45,560 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:20:45,560 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:20:45,560 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:20:45,561 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:20:45,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741930_1106 (size=443) 2024-11-25T08:20:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T08:20:45,971 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:20:45,976 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:20:45,976 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_1732522844576_ns3_test-17325227876632 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:20:45,978 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:20:45,978 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-25T08:20:45,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732522844576_ns3_test-17325227876632 table=ns3:test-17325227876632 type=FLUSH ttl=0 } in 1.3930 sec 2024-11-25T08:20:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-25T08:20:46,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17325227876632 completed 2024-11-25T08:20:46,736 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732522831552 2024-11-25T08:20:46,736 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-25T08:20:46,736 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-25T08:20:46,736 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732522831552_backup:system_bulk 2024-11-25T08:20:46,736 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732522835918_backup_system_bulk to hdfs://localhost:42243/backupUT/backup_1732522831552/backup/system_bulk/ 2024-11-25T08:20:46,736 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-25T08:20:46,771 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:42243, inputRoot=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:20:46,771 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1328517145_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42243/backupUT/backup_1732522831552/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42243/backupUT/backup_1732522831552/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732522835918_backup_system_bulk 2024-11-25T08:20:46,773 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-25T08:20:46,777 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522835918_backup_system_bulk to hdfs://localhost:42243/backupUT/backup_1732522831552/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732522835918_backup_system_bulk 2024-11-25T08:20:46,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741931_1107 (size=76) 2024-11-25T08:20:46,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741932_1108 (size=702) 2024-11-25T08:20:47,188 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:47,189 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:47,189 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,179 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-2777990839972417819.jar 2024-11-25T08:20:48,179 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,179 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,246 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-8637417953403696421.jar 2024-11-25T08:20:48,246 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,246 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,247 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,247 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,247 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,247 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:48,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:20:48,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:20:48,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:20:48,248 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:20:48,249 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:20:48,249 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:20:48,249 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:20:48,250 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:20:48,250 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:20:48,250 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:20:48,250 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:20:48,251 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:48,251 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:48,251 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:48,251 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:48,252 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:48,252 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:48,252 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:48,293 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:20:48,308 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:20:48,315 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732522835918_backup_system_bulk' hfile list 2024-11-25T08:20:48,408 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local53285206_0005_895d3c07-7ef0-4646-a34c-2612e1127d16/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-25T08:20:48,408 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local53285206_0005_895d3c07-7ef0-4646-a34c-2612e1127d16/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-25T08:20:49,449 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-25T08:20:49,450 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-25T08:20:49,455 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732522835918_backup_system_bulk 2024-11-25T08:20:49,455 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732522835918_backup_system_bulk finished. 2024-11-25T08:20:49,455 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732522831552_ns4:test-17325227876633 2024-11-25T08:20:49,455 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732522838076_ns4_test-17325227876633 to hdfs://localhost:42243/backupUT/backup_1732522831552/ns4/test-17325227876633/ 2024-11-25T08:20:49,455 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-25T08:20:49,486 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:42243, inputRoot=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:20:49,486 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1328517145_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42243/backupUT/backup_1732522831552/ns4/test-17325227876633, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42243/backupUT/backup_1732522831552/ns4/test-17325227876633/.hbase-snapshot/.tmp/snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:20:49,488 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-25T08:20:49,492 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522838076_ns4_test-17325227876633 to hdfs://localhost:42243/backupUT/backup_1732522831552/ns4/test-17325227876633/.hbase-snapshot/.tmp/snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:20:49,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741933_1109 (size=86) 2024-11-25T08:20:49,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741934_1110 (size=443) 2024-11-25T08:20:49,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:49,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:49,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,894 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-16946466955862279417.jar 2024-11-25T08:20:50,894 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,895 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,960 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-7535023708969354060.jar 2024-11-25T08:20:50,961 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,961 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,961 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,962 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,962 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,962 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:50,962 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:20:50,963 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:20:50,963 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:20:50,963 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:20:50,963 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:20:50,964 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:20:50,964 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:20:50,964 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:20:50,964 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:20:50,964 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:20:50,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:20:50,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:50,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:50,965 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:50,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:50,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:50,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:50,966 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:51,007 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:20:51,020 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:20:51,026 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732522838076_ns4_test-17325227876633' hfile list 2024-11-25T08:20:51,118 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local1610384464_0006_69e8bf54-5452-40e1-8b1f-f1279d6c2004/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-25T08:20:51,118 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local1610384464_0006_69e8bf54-5452-40e1-8b1f-f1279d6c2004/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-25T08:20:52,158 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-25T08:20:52,159 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-25T08:20:52,163 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:20:52,163 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732522838076_ns4_test-17325227876633 finished. 2024-11-25T08:20:52,163 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732522831552_ns2:test-17325227876631 2024-11-25T08:20:52,163 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732522840246_ns2_test-17325227876631 to hdfs://localhost:42243/backupUT/backup_1732522831552/ns2/test-17325227876631/ 2024-11-25T08:20:52,163 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-25T08:20:52,194 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:42243, inputRoot=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:20:52,194 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1328517145_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42243/backupUT/backup_1732522831552/ns2/test-17325227876631, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42243/backupUT/backup_1732522831552/ns2/test-17325227876631/.hbase-snapshot/.tmp/snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:52,196 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-25T08:20:52,202 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522840246_ns2_test-17325227876631 to hdfs://localhost:42243/backupUT/backup_1732522831552/ns2/test-17325227876631/.hbase-snapshot/.tmp/snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:52,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741935_1111 (size=86) 2024-11-25T08:20:52,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741936_1112 (size=496) 2024-11-25T08:20:52,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:52,615 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:52,616 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,608 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-577077895531177959.jar 2024-11-25T08:20:53,609 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,609 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,676 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-10543385665577772406.jar 2024-11-25T08:20:53,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,677 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:53,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:20:53,678 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:20:53,679 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:20:53,679 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:20:53,679 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:20:53,679 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:20:53,679 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:20:53,680 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:20:53,680 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:20:53,680 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:20:53,680 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:20:53,681 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:53,681 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:53,681 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:53,681 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:53,682 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:53,682 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:53,682 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:53,725 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:20:53,739 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:20:53,745 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732522840246_ns2_test-17325227876631' hfile list 2024-11-25T08:20:53,748 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-11-25T08:20:53,839 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local240427726_0007_41588eec-3fdd-4ea6-a03c-9f7754df974b/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-25T08:20:53,839 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local240427726_0007_41588eec-3fdd-4ea6-a03c-9f7754df974b/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-25T08:20:53,913 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-25T08:20:53,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741937_1113 (size=8413) 2024-11-25T08:20:54,360 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876 output=hdfs://localhost:42243/backupUT/backup_1732522831552/ns2/test-17325227876631/archive/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/4b485798ffaa4bf699dffe1c33406876 2024-11-25T08:20:54,360 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-25T08:20:54,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741938_1114 (size=5134) 2024-11-25T08:20:54,793 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/a071fb1553b44115828b8af4e4cad1df output=hdfs://localhost:42243/backupUT/backup_1732522831552/ns2/test-17325227876631/archive/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/f/a071fb1553b44115828b8af4e4cad1df 2024-11-25T08:20:54,793 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-11-25T08:20:54,882 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-25T08:20:54,883 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-25T08:20:54,889 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:20:54,889 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732522840246_ns2_test-17325227876631 finished. 2024-11-25T08:20:54,890 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732522831552_test-1732522787663 2024-11-25T08:20:54,890 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732522842406_default_test-1732522787663 to hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/ 2024-11-25T08:20:54,890 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-25T08:20:54,920 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:42243, inputRoot=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:20:54,920 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1328517145_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/.hbase-snapshot/.tmp/snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:54,922 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-25T08:20:54,928 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522842406_default_test-1732522787663 to hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/.hbase-snapshot/.tmp/snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:54,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741939_1115 (size=84) 2024-11-25T08:20:54,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741940_1116 (size=1347) 2024-11-25T08:20:55,341 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:55,341 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:55,342 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,333 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-10500447755580960983.jar 2024-11-25T08:20:56,334 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,334 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,400 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-10780748345205334405.jar 2024-11-25T08:20:56,400 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,400 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,401 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,401 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,401 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,401 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:20:56,401 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:20:56,402 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:20:56,402 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:20:56,402 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:20:56,402 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:20:56,403 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:20:56,403 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:20:56,403 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:20:56,403 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:20:56,404 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:20:56,404 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:20:56,404 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:56,404 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:56,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:56,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:56,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:20:56,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:56,405 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:20:56,447 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:20:56,463 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:20:56,470 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732522842406_default_test-1732522787663' hfile list 2024-11-25T08:20:56,472 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-11-25T08:20:56,567 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local649363038_0008_ccb35999-da70-4b6c-a4cf-c95892de7da7/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-25T08:20:56,567 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local649363038_0008_ccb35999-da70-4b6c-a4cf-c95892de7da7/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-25T08:20:56,640 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-25T08:20:56,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741941_1117 (size=13760) 2024-11-25T08:20:57,085 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/c746e2b059854d03a1f0eef704d4e9eb output=hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/archive/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/c746e2b059854d03a1f0eef704d4e9eb 2024-11-25T08:20:57,085 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-25T08:20:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741942_1118 (size=10796) 2024-11-25T08:20:57,519 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/60394def41e549aca954d45be1e14ed2 output=hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/archive/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/60394def41e549aca954d45be1e14ed2 2024-11-25T08:20:57,519 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-25T08:20:57,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741943_1119 (size=8413) 2024-11-25T08:20:57,960 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b output=hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/archive/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f/a259715a07084e5eaab5cf0e2878b34b 2024-11-25T08:20:57,961 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-25T08:20:57,986 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/mobdir/data/default/test-1732522787663/6f9de7d24d3c4b766b3d2632af4ccf70/mob/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5 to hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/archive/data/default/test-1732522787663/6f9de7d24d3c4b766b3d2632af4ccf70/mob/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5" truncated to max limit (512 characters) 2024-11-25T08:20:57,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741944_1120 (size=5591) 2024-11-25T08:20:58,394 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/mobdir/data/default/test-1732522787663/6f9de7d24d3c4b766b3d2632af4ccf70/mob/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5 output=hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/archive/data/default/test-1732522787663/6f9de7d24d3c4b766b3d2632af4ccf70/mob/d41d8cd98f00b204e9800998ecf8427e2024112564e09e4a7c0d46b283eeb18bc6f727ae_9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:20:58,394 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.012M/sec 2024-11-25T08:20:58,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741945_1121 (size=5484) 2024-11-25T08:20:58,837 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 output=hdfs://localhost:42243/backupUT/backup_1732522831552/default/test-1732522787663/archive/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/mob/9b801d1d7dfa4281823e96a92c96fe02 2024-11-25T08:20:58,838 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-25T08:20:59,610 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-25T08:20:59,611 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-25T08:20:59,619 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:20:59,619 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732522842406_default_test-1732522787663 finished. 2024-11-25T08:20:59,619 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732522831552_ns3:test-17325227876632 2024-11-25T08:20:59,619 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732522844576_ns3_test-17325227876632 to hdfs://localhost:42243/backupUT/backup_1732522831552/ns3/test-17325227876632/ 2024-11-25T08:20:59,619 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-25T08:20:59,649 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:42243, inputRoot=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927 2024-11-25T08:20:59,649 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1328517145_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42243/backupUT/backup_1732522831552/ns3/test-17325227876632, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42243/backupUT/backup_1732522831552/ns3/test-17325227876632/.hbase-snapshot/.tmp/snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:20:59,651 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-25T08:20:59,655 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_1732522844576_ns3_test-17325227876632 to hdfs://localhost:42243/backupUT/backup_1732522831552/ns3/test-17325227876632/.hbase-snapshot/.tmp/snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:20:59,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741946_1122 (size=86) 2024-11-25T08:20:59,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741947_1123 (size=443) 2024-11-25T08:21:00,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:00,067 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:00,068 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,056 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-10198619460742963149.jar 2024-11-25T08:21:01,056 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,057 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,123 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-7607217946550578424.jar 2024-11-25T08:21:01,123 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,123 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,124 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,124 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,124 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,125 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:01,125 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:21:01,125 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:21:01,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:21:01,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:21:01,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:21:01,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:21:01,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:21:01,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:21:01,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:21:01,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:21:01,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:21:01,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:21:01,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:21:01,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:21:01,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:21:01,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-25T08:21:01,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:21:01,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:21:01,170 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:21:01,185 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:21:01,191 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732522844576_ns3_test-17325227876632' hfile list 2024-11-25T08:21:01,283 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local1655153483_0009_3e0e21d3-515d-4fac-9508-39a4040a0be9/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-25T08:21:01,283 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/mapreduce.cluster.local.dir/job_local1655153483_0009_3e0e21d3-515d-4fac-9508-39a4040a0be9/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-25T08:21:02,323 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-25T08:21:02,324 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-25T08:21:02,329 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:21:02,329 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732522844576_ns3_test-17325227876632 finished. 2024-11-25T08:21:02,329 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-25T08:21:02,329 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17325227876633 2024-11-25T08:21:02,329 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17325227876631 2024-11-25T08:21:02,329 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732522787663 2024-11-25T08:21:02,329 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17325227876632 2024-11-25T08:21:02,340 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732522831552 2024-11-25T08:21:02,340 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-25T08:21:02,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741948_1124 (size=209) 2024-11-25T08:21:02,746 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42243/backupUT/backup_1732522831552/.backup.manifest 2024-11-25T08:21:02,746 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732522831552 finished: type=FULL,tablelist=backup:system_bulk;ns4:test-17325227876633;ns2:test-17325227876631;test-1732522787663;ns3:test-17325227876632,targetRootDir=hdfs://localhost:42243/backupUT,startts=1732522835746,completets=1732522862340,bytescopied=0 2024-11-25T08:21:02,746 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-25T08:21:02,746 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732522835918_backup_system_bulk 2024-11-25T08:21:02,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732522835918_backup_system_bulk" type: DISABLED 2024-11-25T08:21:02,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732522835918_backup_system_bulk 2024-11-25T08:21:02,750 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732522835918_backup_system_bulk for backup backup_1732522831552 succeeded. 2024-11-25T08:21:02,750 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:21:02,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732522838076_ns4_test-17325227876633" type: DISABLED 2024-11-25T08:21:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732522838076_ns4_test-17325227876633 2024-11-25T08:21:02,754 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732522838076_ns4_test-17325227876633 for backup backup_1732522831552 succeeded. 2024-11-25T08:21:02,754 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:21:02,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732522840246_ns2_test-17325227876631" type: DISABLED 2024-11-25T08:21:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732522840246_ns2_test-17325227876631 2024-11-25T08:21:02,757 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732522840246_ns2_test-17325227876631 for backup backup_1732522831552 succeeded. 2024-11-25T08:21:02,757 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:21:02,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732522842406_default_test-1732522787663" type: DISABLED 2024-11-25T08:21:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732522842406_default_test-1732522787663 2024-11-25T08:21:02,761 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732522842406_default_test-1732522787663 for backup backup_1732522831552 succeeded. 2024-11-25T08:21:02,761 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:21:02,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732522844576_ns3_test-17325227876632" type: DISABLED 2024-11-25T08:21:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732522844576_ns3_test-17325227876632 2024-11-25T08:21:02,764 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732522844576_ns3_test-17325227876632 for backup backup_1732522831552 succeeded. 2024-11-25T08:21:02,765 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-25T08:21:02,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-25T08:21:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-25T08:21:02,771 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-25T08:21:02,773 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-25T08:21:02,780 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732522831552 completed. 2024-11-25T08:21:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:04,092 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T08:21:07,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,851 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-25T08:21:07,856 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732522867808 2024-11-25T08:21:07,859 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732522831552} 2024-11-25T08:21:07,859 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-25T08:21:07,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:07,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:21:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522867959 (current time:1732522867959). 2024-11-25T08:21:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:21:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-25T08:21:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d43f802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:21:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:21:07,961 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:21:07,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:21:07,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:21:07,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5304c20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:07,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:21:07,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:21:07,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:07,963 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:21:07,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a39a44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:21:07,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:21:07,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:21:07,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:21:07,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:21:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:21:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:07,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:21:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:21:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:21:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-25T08:21:07,970 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:21:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T08:21:07,971 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:21:07,973 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:21:07,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741949_1125 (size=52) 2024-11-25T08:21:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T08:21:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T08:21:08,379 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-25T08:21:08,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909}] 2024-11-25T08:21:08,380 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:08,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43089 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-25T08:21:08,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:08,533 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 66dbbb90362d3aeb47c6625cd227a909 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-25T08:21:08,538 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/9603a67c9e82498a98ab7f835ee7c975 is 158, key is trslm:hdfs://localhost:42243/backupUT\x00ns2:test-17325227876631/meta:log-roll-map/1732522862334/Put/seqid=0 2024-11-25T08:21:08,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741950_1126 (size=6646) 2024-11-25T08:21:08,543 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/9603a67c9e82498a98ab7f835ee7c975 2024-11-25T08:21:08,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/b771b1a837ef43a99bfc13bba2616cc2 is 826, key is session:backup_1732522831552/session:context/1732522862772/Put/seqid=0 2024-11-25T08:21:08,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741951_1127 (size=5928) 2024-11-25T08:21:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T08:21:08,956 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/b771b1a837ef43a99bfc13bba2616cc2 2024-11-25T08:21:08,962 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/9603a67c9e82498a98ab7f835ee7c975 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975 2024-11-25T08:21:08,968 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975, entries=12, sequenceid=42, filesize=6.5 K 2024-11-25T08:21:08,969 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/b771b1a837ef43a99bfc13bba2616cc2 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2 2024-11-25T08:21:08,974 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2, entries=2, sequenceid=42, filesize=5.8 K 2024-11-25T08:21:08,975 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 66dbbb90362d3aeb47c6625cd227a909 in 442ms, sequenceid=42, compaction requested=true 2024-11-25T08:21:08,975 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 66dbbb90362d3aeb47c6625cd227a909: 2024-11-25T08:21:08,975 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. for snapshot_backup_system completed. 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.' region-info for snapshot=snapshot_backup_system 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975] hfiles 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5 for snapshot=snapshot_backup_system 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc for snapshot=snapshot_backup_system 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975 for snapshot=snapshot_backup_system 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2] hfiles 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b for snapshot=snapshot_backup_system 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4 for snapshot=snapshot_backup_system 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06 for snapshot=snapshot_backup_system 2024-11-25T08:21:08,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2 for snapshot=snapshot_backup_system 2024-11-25T08:21:08,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741952_1128 (size=330) 2024-11-25T08:21:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T08:21:09,382 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:09,382 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-25T08:21:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-25T08:21:09,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:09,383 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:09,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-25T08:21:09,385 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-25T08:21:09,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 66dbbb90362d3aeb47c6625cd227a909 in 1.0050 sec 2024-11-25T08:21:09,386 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:21:09,386 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:21:09,386 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-25T08:21:09,387 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-25T08:21:09,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741953_1129 (size=967) 2024-11-25T08:21:09,798 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:21:09,807 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:21:09,807 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/snapshot_backup_system 2024-11-25T08:21:09,809 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:21:09,809 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-25T08:21:09,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8410 sec 2024-11-25T08:21:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T08:21:10,126 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-25T08:21:10,126 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732522867808 started at 1732522870126. 2024-11-25T08:21:10,128 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732522867808 has been started. 2024-11-25T08:21:10,133 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, ns2:test-17325227876631, ns3:test-17325227876632, ns4:test-17325227876633, test-1732522787663] 2024-11-25T08:21:10,137 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732522822074for backupID backup_1732522867808 2024-11-25T08:21:10,137 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-25T08:21:10,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-25T08:21:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-25T08:21:10,138 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-25T08:21:10,138 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-25T08:21:10,138 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-25T08:21:10,138 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-25T08:21:10,139 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,139 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-25T08:21:10,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-25T08:21:10,146 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,147 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-25T08:21:10,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:21:10,147 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,147 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-25T08:21:10,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-25T08:21:10,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-25T08:21:10,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-25T08:21:10,148 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-25T08:21:10,149 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-25T08:21:10,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-25T08:21:10,149 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-25T08:21:10,149 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-25T08:21:10,149 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-25T08:21:10,149 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-25T08:21:10,149 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '5eb3d201e8c9,43089,1732522777179' joining acquired barrier for procedure (rolllog) in zk 2024-11-25T08:21:10,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,152 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-25T08:21:10,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:21:10,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:21:10,152 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-25T08:21:10,152 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-25T08:21:10,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:21:10,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:21:10,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:21:10,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:21:10,154 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '5eb3d201e8c9,43089,1732522777179' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-25T08:21:10,154 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@23c83a2e[Count = 0] remaining members to acquire global barrier 2024-11-25T08:21:10,154 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-25T08:21:10,154 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-25T08:21:10,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:21:10,155 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-25T08:21:10,155 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-25T08:21:10,155 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-25T08:21:10,156 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,156 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-25T08:21:10,156 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-25T08:21:10,156 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,156 INFO [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732522835832 highest: 1732522835832 on 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,156 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.meta:.meta(num 1732522835795) roll requested 2024-11-25T08:21:10,170 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522870156.meta, exclude list is [], retry=0 2024-11-25T08:21:10,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:21:10,174 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522835795.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522870156.meta 2024-11-25T08:21:10,174 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:21:10,174 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522835795.meta is not closed yet, will try archiving it next time 2024-11-25T08:21:10,174 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1:(num 1732522835814) roll requested 2024-11-25T08:21:10,175 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522835795.meta not finished, retry = 0 2024-11-25T08:21:10,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741911_1087 (size=93) 2024-11-25T08:21:10,187 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522870174, exclude list is [], retry=0 2024-11-25T08:21:10,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:21:10,191 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522835814 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522870174 2024-11-25T08:21:10,191 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:21:10,191 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522835814 is not closed yet, will try archiving it next time 2024-11-25T08:21:10,191 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0:(num 1732522835832) roll requested 2024-11-25T08:21:10,191 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522822056 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522822056 2024-11-25T08:21:10,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741912_1088 (size=4710) 2024-11-25T08:21:10,204 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522870191, exclude list is [], retry=0 2024-11-25T08:21:10,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43131,DS-e8d3642a-6513-4e62-84b7-0aaa2923d333,DISK] 2024-11-25T08:21:10,208 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522870191 2024-11-25T08:21:10,208 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36805:36805)] 2024-11-25T08:21:10,208 DEBUG [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 is not closed yet, will try archiving it next time 2024-11-25T08:21:10,208 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522802286 2024-11-25T08:21:10,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741913_1089 (size=653) 2024-11-25T08:21:10,210 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 not finished, retry = 0 2024-11-25T08:21:10,216 DEBUG [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-25T08:21:10,216 INFO [rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732522870191 on 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-25T08:21:10,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:10,233 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-25T08:21:10,233 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '5eb3d201e8c9,43089,1732522777179' in zk 2024-11-25T08:21:10,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,235 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-25T08:21:10,235 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-25T08:21:10,235 DEBUG [member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-25T08:21:10,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:21:10,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:21:10,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:21:10,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:21:10,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:21:10,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:21:10,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:21:10,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '5eb3d201e8c9,43089,1732522777179': 2024-11-25T08:21:10,238 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '5eb3d201e8c9,43089,1732522777179' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-25T08:21:10,238 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-25T08:21:10,238 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-25T08:21:10,238 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-25T08:21:10,238 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-25T08:21:10,238 INFO [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-25T08:21:10,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-25T08:21:10,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,240 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-25T08:21:10,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-25T08:21:10,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-25T08:21:10,241 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-25T08:21:10,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:21:10,241 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:21:10,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-25T08:21:10,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:21:10,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,242 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-25T08:21:10,242 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-25T08:21:10,243 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-25T08:21:10,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-25T08:21:10,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-25T08:21:10,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:10,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:21:10,246 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-25T08:21:10,246 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-25T08:21:10,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-25T08:21:10,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-25T08:21:10,247 DEBUG [(5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-25T08:21:10,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-25T08:21:10,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-25T08:21:10,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-25T08:21:10,247 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-25T08:21:10,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-25T08:21:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-25T08:21:10,258 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {5eb3d201e8c9:43089=1732522822074} newestTimestamps: {5eb3d201e8c9:43089=1732522835832} 2024-11-25T08:21:10,260 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522782479.meta 2024-11-25T08:21:10,260 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522822037.meta 2024-11-25T08:21:10,260 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522835795.meta 2024-11-25T08:21:10,260 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522870156.meta 2024-11-25T08:21:10,260 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 2024-11-25T08:21:10,260 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522870191 2024-11-25T08:21:10,260 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522835814 2024-11-25T08:21:10,260 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522870174 2024-11-25T08:21:10,261 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522802223.meta 2024-11-25T08:21:10,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741957_1133 (size=640) 2024-11-25T08:21:10,276 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522835795.meta to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522835795.meta 2024-11-25T08:21:10,311 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 2024-11-25T08:21:10,671 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:42243/backupUT/backup_1732522867808/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:21:10,671 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-25T08:21:10,673 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-25T08:21:10,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741958_1134 (size=45) 2024-11-25T08:21:11,080 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-25T08:21:11,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741959_1135 (size=390) 2024-11-25T08:21:11,491 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17325227876633 target: hdfs://localhost:42243/backupUT/backup_1732522867808/ns4/test-17325227876633 descriptor: 'ns4:test-17325227876633', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:21:11,491 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-25T08:21:11,494 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17325227876633 2024-11-25T08:21:11,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741960_1136 (size=50) 2024-11-25T08:21:11,900 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17325227876633 2024-11-25T08:21:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741961_1137 (size=364) 2024-11-25T08:21:11,911 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17325227876631 target: hdfs://localhost:42243/backupUT/backup_1732522867808/ns2/test-17325227876631 descriptor: 'ns2:test-17325227876631', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:21:11,911 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-25T08:21:11,913 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17325227876631 2024-11-25T08:21:11,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741962_1138 (size=50) 2024-11-25T08:21:12,318 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17325227876631 2024-11-25T08:21:12,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741963_1139 (size=390) 2024-11-25T08:21:12,730 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17325227876632 target: hdfs://localhost:42243/backupUT/backup_1732522867808/ns3/test-17325227876632 descriptor: 'ns3:test-17325227876632', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:21:12,730 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-25T08:21:12,732 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17325227876632 2024-11-25T08:21:12,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741964_1140 (size=50) 2024-11-25T08:21:13,137 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17325227876632 2024-11-25T08:21:13,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741965_1141 (size=950) 2024-11-25T08:21:13,548 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732522787663 target: hdfs://localhost:42243/backupUT/backup_1732522867808/default/test-1732522787663 descriptor: 'test-1732522787663', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T08:21:13,548 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-25T08:21:13,550 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732522787663 2024-11-25T08:21:13,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741966_1142 (size=53) 2024-11-25T08:21:13,956 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732522787663 2024-11-25T08:21:13,965 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:42243/backupUT/.tmp/backup_1732522867808 from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832;hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522835814 2024-11-25T08:21:13,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2273ec4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:13,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:21:13,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:21:13,967 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:21:13,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:21:13,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:21:13,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d76bf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:13,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:21:13,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:21:13,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:13,969 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:21:13,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c0cc86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:13,969 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T08:21:13,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 5eb3d201e8c9,44851,1732522775045 2024-11-25T08:21:13,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@39b26ce0 2024-11-25T08:21:13,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T08:21:13,971 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T08:21:13,974 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-25T08:21:13,974 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-25T08:21:13,974 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-25T08:21:13,974 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:21:13,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:21:13,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:21:13,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:21:13,977 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-25T08:21:13,977 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17325227876631 2024-11-25T08:21:13,978 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17325227876631,, stopping at row=ns2:test-17325227876631 ,, for max=2147483647 with caching=100 2024-11-25T08:21:13,979 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17325227876631: ns2:test-17325227876631; 2024-11-25T08:21:13,979 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17325227876632 2024-11-25T08:21:13,979 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17325227876632,, stopping at row=ns3:test-17325227876632 ,, for max=2147483647 with caching=100 2024-11-25T08:21:13,981 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17325227876632: ns3:test-17325227876632; 2024-11-25T08:21:13,981 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17325227876633 2024-11-25T08:21:13,981 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17325227876633,, stopping at row=ns4:test-17325227876633 ,, for max=2147483647 with caching=100 2024-11-25T08:21:13,982 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17325227876633: ns4:test-17325227876633; 2024-11-25T08:21:13,983 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732522787663 2024-11-25T08:21:13,983 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732522787663,, stopping at row=test-1732522787663 ,, for max=2147483647 with caching=100 2024-11-25T08:21:13,984 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732522787663: test-1732522787663; 2024-11-25T08:21:13,984 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-25T08:21:13,985 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/hbase-staging/partitions_ef9c0eaa-1121-47cb-b93a-b74f77ba5f21 2024-11-25T08:21:13,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741967_1143 (size=324) 2024-11-25T08:21:14,392 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:14,392 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:14,393 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,375 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-6175865867290578786.jar 2024-11-25T08:21:15,376 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,376 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-9195278497764570247.jar 2024-11-25T08:21:15,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:21:15,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-25T08:21:15,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-25T08:21:15,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-25T08:21:15,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-25T08:21:15,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-25T08:21:15,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-25T08:21:15,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-25T08:21:15,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-25T08:21:15,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-25T08:21:15,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-25T08:21:15,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,513 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-6919862068698785862.jar 2024-11-25T08:21:15,514 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-6919862068698785862.jar 2024-11-25T08:21:15,514 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-25T08:21:15,514 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-6919862068698785862.jar 2024-11-25T08:21:15,514 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-6919862068698785862.jar 2024-11-25T08:21:15,515 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-25T08:21:15,515 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,ns2:test-17325227876631,ns3:test-17325227876632,ns4:test-17325227876633,default:test-1732522787663 2024-11-25T08:21:15,515 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T08:21:15,515 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:21:15,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:15,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:15,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:21:15,516 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-25T08:21:16,380 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9c7bcb575c3e97a1ab0077b07fb801c5, had cached 0 bytes from a total of 38453 2024-11-25T08:21:16,498 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop-16839748515404332853.jar 2024-11-25T08:21:16,510 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-25T08:21:16,524 WARN [Time-limited test {}] mapreduce.JobResourceUploader(481): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2024-11-25T08:21:16,528 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 for WAL files 2024-11-25T08:21:16,529 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1732522867808File hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 does not exist. java.io.FileNotFoundException: File hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[classes/:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[classes/:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[classes/:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T08:21:16,530 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732522867808,startts=1732522870126,failedts=1732522876530,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522835832 does not exist. 2024-11-25T08:21:16,530 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-25T08:21:16,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable backup:system 2024-11-25T08:21:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-11-25T08:21:16,547 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522876546"}]},"ts":"1732522876546"} 2024-11-25T08:21:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T08:21:16,548 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-11-25T08:21:16,548 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-11-25T08:21:16,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-11-25T08:21:16,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, UNASSIGN}] 2024-11-25T08:21:16,554 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, UNASSIGN 2024-11-25T08:21:16,555 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=66dbbb90362d3aeb47c6625cd227a909, regionState=CLOSING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:16,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, UNASSIGN because future has completed 2024-11-25T08:21:16,557 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T08:21:16,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:21:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T08:21:16,709 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:16,709 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T08:21:16,709 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 66dbbb90362d3aeb47c6625cd227a909, disabling compactions & flushes 2024-11-25T08:21:16,709 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:16,710 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:16,710 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. after waiting 0 ms 2024-11-25T08:21:16,710 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:16,710 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 66dbbb90362d3aeb47c6625cd227a909 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-11-25T08:21:16,714 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/0cc6e21d8a54459883c71ba588bd3d7a is 103, key is rslogts:hdfs://localhost:42243/backupUT\x005eb3d201e8c9:43089/meta:rs-log-ts/1732522870232/Put/seqid=0 2024-11-25T08:21:16,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741968_1144 (size=5338) 2024-11-25T08:21:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T08:21:17,119 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/0cc6e21d8a54459883c71ba588bd3d7a 2024-11-25T08:21:17,127 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/d5b839df59964ef2aba0c5cb9514c918 is 303, key is session:backup_1732522867808/session:context/1732522870127/Put/seqid=0 2024-11-25T08:21:17,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741969_1145 (size=5391) 2024-11-25T08:21:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T08:21:17,532 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/d5b839df59964ef2aba0c5cb9514c918 2024-11-25T08:21:17,537 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/0cc6e21d8a54459883c71ba588bd3d7a as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/0cc6e21d8a54459883c71ba588bd3d7a 2024-11-25T08:21:17,542 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/0cc6e21d8a54459883c71ba588bd3d7a, entries=1, sequenceid=47, filesize=5.2 K 2024-11-25T08:21:17,543 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/d5b839df59964ef2aba0c5cb9514c918 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d5b839df59964ef2aba0c5cb9514c918 2024-11-25T08:21:17,549 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d5b839df59964ef2aba0c5cb9514c918, entries=1, sequenceid=47, filesize=5.3 K 2024-11-25T08:21:17,550 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 66dbbb90362d3aeb47c6625cd227a909 in 839ms, sequenceid=47, compaction requested=true 2024-11-25T08:21:17,554 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-11-25T08:21:17,554 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:21:17,555 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:17,555 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 66dbbb90362d3aeb47c6625cd227a909: Waiting for close lock at 1732522876709Running coprocessor pre-close hooks at 1732522876709Disabling compacts and flushes for region at 1732522876709Disabling writes for close at 1732522876710 (+1 ms)Obtaining lock to block concurrent updates at 1732522876710Preparing flush snapshotting stores in 66dbbb90362d3aeb47c6625cd227a909 at 1732522876710Finished memstore snapshotting backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1732522876710Flushing stores of backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. at 1732522876711 (+1 ms)Flushing 66dbbb90362d3aeb47c6625cd227a909/meta: creating writer at 1732522876711Flushing 66dbbb90362d3aeb47c6625cd227a909/meta: appending metadata at 1732522876713 (+2 ms)Flushing 66dbbb90362d3aeb47c6625cd227a909/meta: closing flushed file at 1732522876713Flushing 66dbbb90362d3aeb47c6625cd227a909/session: creating writer at 1732522877124 (+411 ms)Flushing 66dbbb90362d3aeb47c6625cd227a909/session: appending metadata at 1732522877127 (+3 ms)Flushing 66dbbb90362d3aeb47c6625cd227a909/session: closing flushed file at 1732522877127Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4acb8a27: reopening flushed file at 1732522877537 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@346203fc: reopening flushed file at 1732522877543 (+6 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 66dbbb90362d3aeb47c6625cd227a909 in 839ms, sequenceid=47, compaction requested=true at 1732522877550 (+7 ms)Writing region close event to WAL at 1732522877551 (+1 ms)Running coprocessor post-close hooks at 1732522877554 (+3 ms)Closed at 1732522877555 (+1 ms) 2024-11-25T08:21:17,557 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:17,557 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=66dbbb90362d3aeb47c6625cd227a909, regionState=CLOSED 2024-11-25T08:21:17,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:21:17,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-25T08:21:17,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179 in 1.0030 sec 2024-11-25T08:21:17,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-25T08:21:17,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, UNASSIGN in 1.0080 sec 2024-11-25T08:21:17,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-25T08:21:17,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0150 sec 2024-11-25T08:21:17,566 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522877566"}]},"ts":"1732522877566"} 2024-11-25T08:21:17,568 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-11-25T08:21:17,568 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-11-25T08:21:17,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0290 sec 2024-11-25T08:21:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T08:21:17,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-11-25T08:21:17,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732522877694 2024-11-25T08:21:17,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:21:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732522877695 (current time:1732522877695). 2024-11-25T08:21:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-25T08:21:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1732522877694 VERSION not specified, setting to 2 2024-11-25T08:21:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333a72ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ClusterIdFetcher(90): Going to request 5eb3d201e8c9,44851,-1 for getting cluster id 2024-11-25T08:21:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T08:21:17,697 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20ef2d0a-a99b-4ad7-a633-df33eb0156be' 2024-11-25T08:21:17,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T08:21:17,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20ef2d0a-a99b-4ad7-a633-df33eb0156be" 2024-11-25T08:21:17,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2cfe81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:17,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5eb3d201e8c9,44851,-1] 2024-11-25T08:21:17,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T08:21:17,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:17,698 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T08:21:17,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e01c4f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-25T08:21:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T08:21:17,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5eb3d201e8c9,43089,1732522777179, seqNum=-1] 2024-11-25T08:21:17,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T08:21:17,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T08:21:17,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851. 2024-11-25T08:21:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T08:21:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:17,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:21:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-25T08:21:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:21:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-25T08:21:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T08:21:17,704 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-25T08:21:17,705 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-25T08:21:17,707 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-25T08:21:17,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741970_1146 (size=81) 2024-11-25T08:21:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T08:21:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T08:21:18,113 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-11-25T08:21:18,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-25T08:21:18,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-25T08:21:18,114 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:21:18,116 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/0cc6e21d8a54459883c71ba588bd3d7a, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5] hfiles 2024-11-25T08:21:18,116 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/0cc6e21d8a54459883c71ba588bd3d7a 2024-11-25T08:21:18,116 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975 2024-11-25T08:21:18,116 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc 2024-11-25T08:21:18,116 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5 2024-11-25T08:21:18,116 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:21:18,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d5b839df59964ef2aba0c5cb9514c918, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06] hfiles 2024-11-25T08:21:18,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4 2024-11-25T08:21:18,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b 2024-11-25T08:21:18,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2 2024-11-25T08:21:18,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d5b839df59964ef2aba0c5cb9514c918 2024-11-25T08:21:18,118 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06 2024-11-25T08:21:18,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741971_1147 (size=409) 2024-11-25T08:21:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T08:21:18,524 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 66dbbb90362d3aeb47c6625cd227a909, NAME => 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-11-25T08:21:18,525 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-25T08:21:18,526 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-25T08:21:18,526 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1732522877694 2024-11-25T08:21:18,527 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732522877694 2024-11-25T08:21:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741972_1148 (size=1046) 2024-11-25T08:21:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T08:21:18,937 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-25T08:21:18,946 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-25T08:21:18,946 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732522877694 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1732522877694 2024-11-25T08:21:18,948 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-25T08:21:18,948 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-25T08:21:18,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732522877694 table=backup:system type=FLUSH ttl=0 } in 1.2460 sec 2024-11-25T08:21:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T08:21:19,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-25T08:21:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732522867959 type: FLUSH version: 2 ttl: 0 ) 2024-11-25T08:21:19,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-11-25T08:21:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T08:21:19,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741973_1149 (size=635) 2024-11-25T08:21:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T08:21:20,153 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 74322db2b59a5db2ea28349c44aa1b80, had cached 0 bytes from a total of 13547 2024-11-25T08:21:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T08:21:20,289 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-11-25T08:21:20,289 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-11-25T08:21:20,291 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-25T08:21:20,297 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732522867959 type: FLUSH version: 2 ttl: 0 2024-11-25T08:21:20,297 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system 2024-11-25T08:21:20,300 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-11-25T08:21:20,300 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(229): region to restore: 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,304 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:21:20,307 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/0cc6e21d8a54459883c71ba588bd3d7a to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/0cc6e21d8a54459883c71ba588bd3d7a 2024-11-25T08:21:20,308 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:21:20,311 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d5b839df59964ef2aba0c5cb9514c918 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d5b839df59964ef2aba0c5cb9514c918 2024-11-25T08:21:20,311 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732522867959 type: FLUSH version: 2 ttl: 0 2024-11-25T08:21:20,313 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732522880312"}]},"ts":"9223372036854775807"} 2024-11-25T08:21:20,316 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T08:21:20,316 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 66dbbb90362d3aeb47c6625cd227a909, NAME => 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T08:21:20,316 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732522880313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732522880313"}]},"ts":"1732522880313"} 2024-11-25T08:21:20,318 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T08:21:20,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-11-25T08:21:20,318 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => 66dbbb90362d3aeb47c6625cd227a909, NAME => 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T08:21:20,318 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-11-25T08:21:20,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732522867959 type: FLUSH version: 2 ttl: 0 ) in 449 msec 2024-11-25T08:21:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-25T08:21:20,506 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732522877694 2024-11-25T08:21:20,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "hbase-failsafe-snapshot_backup_system-1732522877694" type: DISABLED 2024-11-25T08:21:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1732522877694 2024-11-25T08:21:20,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.2 enable backup:system 2024-11-25T08:21:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-11-25T08:21:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T08:21:20,522 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-11-25T08:21:20,523 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522880522"}]},"ts":"1732522880522"} 2024-11-25T08:21:20,524 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-25T08:21:20,525 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-11-25T08:21:20,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN}] 2024-11-25T08:21:20,526 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN 2024-11-25T08:21:20,527 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-11-25T08:21:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T08:21:20,678 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=66dbbb90362d3aeb47c6625cd227a909, regionState=OPENING, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:20,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN because future has completed 2024-11-25T08:21:20,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179}] 2024-11-25T08:21:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T08:21:20,836 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:20,837 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => 66dbbb90362d3aeb47c6625cd227a909, NAME => 'backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.', STARTKEY => '', ENDKEY => ''} 2024-11-25T08:21:20,837 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-25T08:21:20,837 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,837 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T08:21:20,838 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,838 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,839 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,840 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66dbbb90362d3aeb47c6625cd227a909 columnFamilyName meta 2024-11-25T08:21:20,840 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:21:20,848 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975 2024-11-25T08:21:20,852 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc 2024-11-25T08:21:20,857 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5 2024-11-25T08:21:20,857 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.HStore(327): Store=66dbbb90362d3aeb47c6625cd227a909/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:21:20,857 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,858 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66dbbb90362d3aeb47c6625cd227a909 columnFamilyName session 2024-11-25T08:21:20,858 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T08:21:20,865 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4 2024-11-25T08:21:20,869 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b 2024-11-25T08:21:20,873 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2 2024-11-25T08:21:20,877 DEBUG [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06 2024-11-25T08:21:20,878 INFO [StoreOpener-66dbbb90362d3aeb47c6625cd227a909-1 {}] regionserver.HStore(327): Store=66dbbb90362d3aeb47c6625cd227a909/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T08:21:20,878 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,878 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,880 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,880 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,880 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,881 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T08:21:20,882 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,883 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened 66dbbb90362d3aeb47c6625cd227a909; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11775039200, jitterRate=0.09663598239421844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T08:21:20,883 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:20,884 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for 66dbbb90362d3aeb47c6625cd227a909: Running coprocessor pre-open hook at 1732522880838Writing region info on filesystem at 1732522880838Initializing all the Stores at 1732522880839 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522880839Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732522880839Cleaning up temporary data from old regions at 1732522880880 (+41 ms)Running coprocessor post-open hooks at 1732522880883 (+3 ms)Region opened successfully at 1732522880884 (+1 ms) 2024-11-25T08:21:20,885 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., pid=70, masterSystemTime=1732522880832 2024-11-25T08:21:20,886 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 66dbbb90362d3aeb47c6625cd227a909:meta, priority=-2147483648, current under compaction store size is 1 2024-11-25T08:21:20,889 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T08:21:20,889 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T08:21:20,889 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 66dbbb90362d3aeb47c6625cd227a909:session, priority=-2147483648, current under compaction store size is 2 2024-11-25T08:21:20,889 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T08:21:20,889 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T08:21:20,891 DEBUG [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:20,892 INFO [RS_OPEN_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:20,892 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T08:21:20,892 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T08:21:20,892 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=66dbbb90362d3aeb47c6625cd227a909, regionState=OPEN, openSeqNum=51, regionLocation=5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:20,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179 because future has completed 2024-11-25T08:21:20,894 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.HStore(1541): 66dbbb90362d3aeb47c6625cd227a909/session is initiating minor compaction (all files) 2024-11-25T08:21:20,894 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.HStore(1541): 66dbbb90362d3aeb47c6625cd227a909/meta is initiating minor compaction (all files) 2024-11-25T08:21:20,895 INFO [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66dbbb90362d3aeb47c6625cd227a909/session in backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:20,895 INFO [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66dbbb90362d3aeb47c6625cd227a909/meta in backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:20,895 INFO [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2] into tmpdir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp, totalSize=22.7 K 2024-11-25T08:21:20,895 INFO [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975] into tmpdir=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp, totalSize=17.9 K 2024-11-25T08:21:20,896 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] compactions.Compactor(225): Compacting cea31546a83545cfb8ba5d035493ebb5, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732522802356 2024-11-25T08:21:20,896 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fac197fb6374a9ea5dfe210eeb91f1b, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732522799818 2024-11-25T08:21:20,897 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] compactions.Compactor(225): Compacting 08222de5beb242849698271cb3775eb4, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732522816167 2024-11-25T08:21:20,897 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] compactions.Compactor(225): Compacting a618dcdf0f4b4c91889239c79c4576bc, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732522822117 2024-11-25T08:21:20,897 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c07d95a1c34d869ecba7f8f3cd5b06, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732522830843 2024-11-25T08:21:20,897 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9603a67c9e82498a98ab7f835ee7c975, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732522835870 2024-11-25T08:21:20,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-25T08:21:20,898 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] compactions.Compactor(225): Compacting b771b1a837ef43a99bfc13bba2616cc2, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732522862772 2024-11-25T08:21:20,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 66dbbb90362d3aeb47c6625cd227a909, server=5eb3d201e8c9,43089,1732522777179 in 215 msec 2024-11-25T08:21:20,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-25T08:21:20,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=66dbbb90362d3aeb47c6625cd227a909, ASSIGN in 373 msec 2024-11-25T08:21:20,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732522880902"}]},"ts":"1732522880902"} 2024-11-25T08:21:20,904 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-25T08:21:20,904 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-11-25T08:21:20,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 389 msec 2024-11-25T08:21:20,934 INFO [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66dbbb90362d3aeb47c6625cd227a909#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T08:21:20,934 INFO [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66dbbb90362d3aeb47c6625cd227a909#meta#compaction#16 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T08:21:20,935 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/7a95c453f3a644988d3cb3c2eb1134d8 is 1245, key is session:backup_1732522831449/session:context/1732522831547/Put/seqid=0 2024-11-25T08:21:20,935 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/20b69c9440a847a285341d230e3c2125 is 158, key is trslm:hdfs://localhost:42243/backupUT\x00ns2:test-17325227876631/meta:log-roll-map/1732522862334/Put/seqid=0 2024-11-25T08:21:20,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741974_1150 (size=8073) 2024-11-25T08:21:20,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741975_1151 (size=6749) 2024-11-25T08:21:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T08:21:21,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-11-25T08:21:21,146 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-11-25T08:21:21,146 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-25T08:21:21,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-25T08:21:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44851 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-25T08:21:21,154 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-25T08:21:21,154 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-11-25T08:21:21,156 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-25T08:21:21,162 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732522867808 failed. 2024-11-25T08:21:21,162 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T08:21:21,162 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:21:21,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:21,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:21,162 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:21:21,189 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=627 (was 611) Potentially hanging thread: member: '5eb3d201e8c9,43089,1732522777179' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-501276170_20 at /127.0.0.1:46544 [Receiving block BP-814101347-172.17.0.2-1732522768861:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814101347-172.17.0.2-1732522768861:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;5eb3d201e8c9:43089-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814101347-172.17.0.2-1732522768861:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-501276170_20 at /127.0.0.1:46564 [Receiving block BP-814101347-172.17.0.2-1732522768861:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/20b69c9440a847a285341d230e3c2125 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (5eb3d201e8c9,44851,1732522775045)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-501276170_20 at /127.0.0.1:46558 [Receiving block BP-814101347-172.17.0.2-1732522768861:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/7a95c453f3a644988d3cb3c2eb1134d8 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2101621293_20 at /127.0.0.1:33650 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-814101347-172.17.0.2-1732522768861:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: rs(5eb3d201e8c9,43089,1732522777179)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/5eb3d201e8c9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=637 (was 631) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 375), ProcessCount=15 (was 15), AvailableMemoryMB=5402 (was 5916) 2024-11-25T08:21:21,191 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=627 is superior to 500 2024-11-25T08:21:21,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T08:21:21,198 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T08:21:21,198 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:21:21,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:21,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:21,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:21:21,198 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T08:21:21,198 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2009745068, stopped=false 2024-11-25T08:21:21,199 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5eb3d201e8c9,44851,1732522775045 2024-11-25T08:21:21,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-25T08:21:21,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-25T08:21:21,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:21:21,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:21:21,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T08:21:21,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-25T08:21:21,201 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T08:21:21,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-25T08:21:21,201 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:21:21,202 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:21,202 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5eb3d201e8c9,43089,1732522777179' ***** 2024-11-25T08:21:21,202 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T08:21:21,202 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-25T08:21:21,207 INFO [RS:0;5eb3d201e8c9:43089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@2f5ab22c{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-25T08:21:21,210 INFO [RS:0;5eb3d201e8c9:43089 {}] server.AbstractConnector(383): Stopped ServerConnector@467e5a93{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-25T08:21:21,210 INFO [RS:0;5eb3d201e8c9:43089 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T08:21:21,211 INFO [RS:0;5eb3d201e8c9:43089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@19e753c2{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-25T08:21:21,211 INFO [RS:0;5eb3d201e8c9:43089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@e73d64a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,STOPPED} 2024-11-25T08:21:21,213 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T08:21:21,213 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T08:21:21,213 INFO [RS:0;5eb3d201e8c9:43089 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T08:21:21,213 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-25T08:21:21,213 INFO [RS:0;5eb3d201e8c9:43089 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(3091): Received CLOSE for 74322db2b59a5db2ea28349c44aa1b80 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(3091): Received CLOSE for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(3091): Received CLOSE for daf94d1506be6ea0f96237e4c58ae364 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(3091): Received CLOSE for 53385123a893b8dd1a5dbd90b0513715 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(3091): Received CLOSE for 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(3091): Received CLOSE for 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(959): stopping server 5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T08:21:21,214 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 74322db2b59a5db2ea28349c44aa1b80, disabling compactions & flushes 2024-11-25T08:21:21,214 INFO [RS:0;5eb3d201e8c9:43089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5eb3d201e8c9:43089. 2024-11-25T08:21:21,214 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:21:21,214 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:21:21,214 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. after waiting 0 ms 2024-11-25T08:21:21,214 DEBUG [RS:0;5eb3d201e8c9:43089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:21:21,214 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:21:21,214 DEBUG [RS:0;5eb3d201e8c9:43089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:21,215 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T08:21:21,219 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns2/test-17325227876631/74322db2b59a5db2ea28349c44aa1b80/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-25T08:21:21,220 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:21:21,220 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:21:21,220 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 74322db2b59a5db2ea28349c44aa1b80: Waiting for close lock at 1732522881214Running coprocessor pre-close hooks at 1732522881214Disabling compacts and flushes for region at 1732522881214Disabling writes for close at 1732522881214Writing region close event to WAL at 1732522881215 (+1 ms)Running coprocessor post-close hooks at 1732522881220 (+5 ms)Closed at 1732522881220 2024-11-25T08:21:21,220 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80. 2024-11-25T08:21:21,220 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 44a4d8f1b8ed294e1ddee3eb2e0fe2e9, disabling compactions & flushes 2024-11-25T08:21:21,220 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:21:21,220 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:21:21,220 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. after waiting 0 ms 2024-11-25T08:21:21,220 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:21:21,224 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns4/test-17325227876633/44a4d8f1b8ed294e1ddee3eb2e0fe2e9/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-25T08:21:21,224 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:21:21,224 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:21:21,224 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 44a4d8f1b8ed294e1ddee3eb2e0fe2e9: Waiting for close lock at 1732522881220Running coprocessor pre-close hooks at 1732522881220Disabling compacts and flushes for region at 1732522881220Disabling writes for close at 1732522881220Writing region close event to WAL at 1732522881221 (+1 ms)Running coprocessor post-close hooks at 1732522881224 (+3 ms)Closed at 1732522881224 2024-11-25T08:21:21,224 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17325227876633,,1732522792439.44a4d8f1b8ed294e1ddee3eb2e0fe2e9. 2024-11-25T08:21:21,225 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing daf94d1506be6ea0f96237e4c58ae364, disabling compactions & flushes 2024-11-25T08:21:21,225 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:21:21,225 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:21:21,225 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. after waiting 0 ms 2024-11-25T08:21:21,225 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:21:21,228 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system_bulk/daf94d1506be6ea0f96237e4c58ae364/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-25T08:21:21,228 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:21:21,228 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:21:21,229 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for daf94d1506be6ea0f96237e4c58ae364: Waiting for close lock at 1732522881225Running coprocessor pre-close hooks at 1732522881225Disabling compacts and flushes for region at 1732522881225Disabling writes for close at 1732522881225Writing region close event to WAL at 1732522881225Running coprocessor post-close hooks at 1732522881228 (+3 ms)Closed at 1732522881228 2024-11-25T08:21:21,229 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1732522798632.daf94d1506be6ea0f96237e4c58ae364. 2024-11-25T08:21:21,229 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 53385123a893b8dd1a5dbd90b0513715, disabling compactions & flushes 2024-11-25T08:21:21,229 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:21:21,229 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:21:21,229 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. after waiting 0 ms 2024-11-25T08:21:21,229 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:21:21,232 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/ns3/test-17325227876632/53385123a893b8dd1a5dbd90b0513715/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-25T08:21:21,233 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:21:21,233 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:21:21,233 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 53385123a893b8dd1a5dbd90b0513715: Waiting for close lock at 1732522881229Running coprocessor pre-close hooks at 1732522881229Disabling compacts and flushes for region at 1732522881229Disabling writes for close at 1732522881229Writing region close event to WAL at 1732522881229Running coprocessor post-close hooks at 1732522881233 (+4 ms)Closed at 1732522881233 2024-11-25T08:21:21,233 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17325227876632,,1732522791284.53385123a893b8dd1a5dbd90b0513715. 2024-11-25T08:21:21,233 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9c7bcb575c3e97a1ab0077b07fb801c5, disabling compactions & flushes 2024-11-25T08:21:21,233 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:21:21,233 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:21:21,233 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. after waiting 0 ms 2024-11-25T08:21:21,233 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:21:21,234 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9c7bcb575c3e97a1ab0077b07fb801c5 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-25T08:21:21,250 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f2/4b41c604986a4344a1296c8b7ce66c23 is 58, key is row-test-1732522787663-2-0/f2:q1/1732522862799/Put/seqid=0 2024-11-25T08:21:21,252 INFO [regionserver/5eb3d201e8c9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T08:21:21,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741976_1152 (size=5464) 2024-11-25T08:21:21,349 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/7a95c453f3a644988d3cb3c2eb1134d8 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7a95c453f3a644988d3cb3c2eb1134d8 2024-11-25T08:21:21,349 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/meta/20b69c9440a847a285341d230e3c2125 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/20b69c9440a847a285341d230e3c2125 2024-11-25T08:21:21,364 INFO [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66dbbb90362d3aeb47c6625cd227a909/session of 66dbbb90362d3aeb47c6625cd227a909 into 7a95c453f3a644988d3cb3c2eb1134d8(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T08:21:21,364 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66dbbb90362d3aeb47c6625cd227a909: 2024-11-25T08:21:21,364 INFO [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66dbbb90362d3aeb47c6625cd227a909/meta of 66dbbb90362d3aeb47c6625cd227a909 into 20b69c9440a847a285341d230e3c2125(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T08:21:21,364 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66dbbb90362d3aeb47c6625cd227a909: 2024-11-25T08:21:21,366 INFO [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., storeName=66dbbb90362d3aeb47c6625cd227a909/session, priority=12, startTime=1732522880889; duration=0sec 2024-11-25T08:21:21,366 INFO [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., storeName=66dbbb90362d3aeb47c6625cd227a909/meta, priority=13, startTime=1732522880886; duration=0sec 2024-11-25T08:21:21,367 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T08:21:21,367 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T08:21:21,367 DEBUG [RS:0;5eb3d201e8c9:43089-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66dbbb90362d3aeb47c6625cd227a909:session 2024-11-25T08:21:21,367 DEBUG [RS:0;5eb3d201e8c9:43089-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66dbbb90362d3aeb47c6625cd227a909:meta 2024-11-25T08:21:21,367 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T08:21:21,367 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T08:21:21,367 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T08:21:21,368 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-25T08:21:21,368 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 9c7bcb575c3e97a1ab0077b07fb801c5=test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., 66dbbb90362d3aeb47c6625cd227a909=backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.} 2024-11-25T08:21:21,368 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T08:21:21,368 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T08:21:21,368 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T08:21:21,368 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T08:21:21,368 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T08:21:21,368 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 66dbbb90362d3aeb47c6625cd227a909, 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:21:21,368 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-11-25T08:21:21,392 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/info/8947e14d867e4f949d53c7d1abac4da9 is 159, key is ns2:test-17325227876631,,1732522788994.74322db2b59a5db2ea28349c44aa1b80./info:regioninfo/1732522790175/Put/seqid=0 2024-11-25T08:21:21,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741977_1153 (size=14126) 2024-11-25T08:21:21,438 INFO [regionserver/5eb3d201e8c9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T08:21:21,439 INFO [regionserver/5eb3d201e8c9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T08:21:21,568 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 66dbbb90362d3aeb47c6625cd227a909, 9c7bcb575c3e97a1ab0077b07fb801c5 2024-11-25T08:21:21,655 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f2/4b41c604986a4344a1296c8b7ce66c23 2024-11-25T08:21:21,662 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/.tmp/f2/4b41c604986a4344a1296c8b7ce66c23 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f2/4b41c604986a4344a1296c8b7ce66c23 2024-11-25T08:21:21,668 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/f2/4b41c604986a4344a1296c8b7ce66c23, entries=7, sequenceid=348, filesize=5.3 K 2024-11-25T08:21:21,669 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 9c7bcb575c3e97a1ab0077b07fb801c5 in 436ms, sequenceid=348, compaction requested=false 2024-11-25T08:21:21,673 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/default/test-1732522787663/9c7bcb575c3e97a1ab0077b07fb801c5/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-25T08:21:21,674 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:21:21,674 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:21:21,674 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9c7bcb575c3e97a1ab0077b07fb801c5: Waiting for close lock at 1732522881233Running coprocessor pre-close hooks at 1732522881233Disabling compacts and flushes for region at 1732522881233Disabling writes for close at 1732522881233Obtaining lock to block concurrent updates at 1732522881234 (+1 ms)Preparing flush snapshotting stores in 9c7bcb575c3e97a1ab0077b07fb801c5 at 1732522881234Finished memstore snapshotting test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1732522881234Flushing stores of test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. at 1732522881234Flushing 9c7bcb575c3e97a1ab0077b07fb801c5/f2: creating writer at 1732522881235 (+1 ms)Flushing 9c7bcb575c3e97a1ab0077b07fb801c5/f2: appending metadata at 1732522881249 (+14 ms)Flushing 9c7bcb575c3e97a1ab0077b07fb801c5/f2: closing flushed file at 1732522881250 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59faf6f0: reopening flushed file at 1732522881662 (+412 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 9c7bcb575c3e97a1ab0077b07fb801c5 in 436ms, sequenceid=348, compaction requested=false at 1732522881669 (+7 ms)Writing region close event to WAL at 1732522881670 (+1 ms)Running coprocessor post-close hooks at 1732522881673 (+3 ms)Closed at 1732522881674 (+1 ms) 2024-11-25T08:21:21,674 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1732522787663,,1732522788153.9c7bcb575c3e97a1ab0077b07fb801c5. 2024-11-25T08:21:21,674 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66dbbb90362d3aeb47c6625cd227a909, disabling compactions & flushes 2024-11-25T08:21:21,674 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:21,674 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:21,674 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. after waiting 0 ms 2024-11-25T08:21:21,674 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:21,674 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 66dbbb90362d3aeb47c6625cd227a909 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-11-25T08:21:21,678 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/99b55429d46a420e911f335458b9ae6a is 554, key is session:backup_1732522867808/session:context/1732522881155/Put/seqid=0 2024-11-25T08:21:21,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741978_1154 (size=5655) 2024-11-25T08:21:21,769 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:21,798 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/info/8947e14d867e4f949d53c7d1abac4da9 2024-11-25T08:21:21,819 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/ns/39886e7e630949a48f456817ceac58db is 43, key is default/ns:d/1732522782736/Put/seqid=0 2024-11-25T08:21:21,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741979_1155 (size=5345) 2024-11-25T08:21:21,969 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 66dbbb90362d3aeb47c6625cd227a909 2024-11-25T08:21:22,083 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/99b55429d46a420e911f335458b9ae6a 2024-11-25T08:21:22,088 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/.tmp/session/99b55429d46a420e911f335458b9ae6a as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/99b55429d46a420e911f335458b9ae6a 2024-11-25T08:21:22,094 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/99b55429d46a420e911f335458b9ae6a, entries=2, sequenceid=57, filesize=5.5 K 2024-11-25T08:21:22,095 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 66dbbb90362d3aeb47c6625cd227a909 in 421ms, sequenceid=57, compaction requested=false 2024-11-25T08:21:22,095 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975] to archive 2024-11-25T08:21:22,096 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T08:21:22,099 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/cea31546a83545cfb8ba5d035493ebb5 2024-11-25T08:21:22,100 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/a618dcdf0f4b4c91889239c79c4576bc 2024-11-25T08:21:22,101 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/meta/9603a67c9e82498a98ab7f835ee7c975 2024-11-25T08:21:22,107 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=5eb3d201e8c9:44851 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-25T08:21:22,108 WARN [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [cea31546a83545cfb8ba5d035493ebb5=5914, a618dcdf0f4b4c91889239c79c4576bc=5745, 9603a67c9e82498a98ab7f835ee7c975=6646] 2024-11-25T08:21:22,108 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2] to archive 2024-11-25T08:21:22,109 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T08:21:22,111 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/7fac197fb6374a9ea5dfe210eeb91f1b 2024-11-25T08:21:22,113 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/08222de5beb242849698271cb3775eb4 2024-11-25T08:21:22,114 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/d7c07d95a1c34d869ecba7f8f3cd5b06 2024-11-25T08:21:22,115 DEBUG [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2 to hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/archive/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/session/b771b1a837ef43a99bfc13bba2616cc2 2024-11-25T08:21:22,115 WARN [StoreCloser-backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [7fac197fb6374a9ea5dfe210eeb91f1b=5049, 08222de5beb242849698271cb3775eb4=5535, d7c07d95a1c34d869ecba7f8f3cd5b06=6682, b771b1a837ef43a99bfc13bba2616cc2=5928] 2024-11-25T08:21:22,119 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/backup/system/66dbbb90362d3aeb47c6625cd227a909/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-11-25T08:21:22,119 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:21:22,119 INFO [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:22,120 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66dbbb90362d3aeb47c6625cd227a909: Waiting for close lock at 1732522881674Running coprocessor pre-close hooks at 1732522881674Disabling compacts and flushes for region at 1732522881674Disabling writes for close at 1732522881674Obtaining lock to block concurrent updates at 1732522881674Preparing flush snapshotting stores in 66dbbb90362d3aeb47c6625cd227a909 at 1732522881674Finished memstore snapshotting backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1732522881674Flushing stores of backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. at 1732522881675 (+1 ms)Flushing 66dbbb90362d3aeb47c6625cd227a909/session: creating writer at 1732522881675Flushing 66dbbb90362d3aeb47c6625cd227a909/session: appending metadata at 1732522881677 (+2 ms)Flushing 66dbbb90362d3aeb47c6625cd227a909/session: closing flushed file at 1732522881677Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e34fd78: reopening flushed file at 1732522882088 (+411 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 66dbbb90362d3aeb47c6625cd227a909 in 421ms, sequenceid=57, compaction requested=false at 1732522882095 (+7 ms)Writing region close event to WAL at 1732522882115 (+20 ms)Running coprocessor post-close hooks at 1732522882119 (+4 ms)Closed at 1732522882119 2024-11-25T08:21:22,120 DEBUG [RS_CLOSE_REGION-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1732522796444.66dbbb90362d3aeb47c6625cd227a909. 2024-11-25T08:21:22,169 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T08:21:22,224 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/ns/39886e7e630949a48f456817ceac58db 2024-11-25T08:21:22,245 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/table/cde2ec912f8b45d195a90bc74d67dca7 is 59, key is ns2:test-17325227876631/table:state/1732522790189/Put/seqid=0 2024-11-25T08:21:22,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741980_1156 (size=5866) 2024-11-25T08:21:22,369 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T08:21:22,369 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T08:21:22,369 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T08:21:22,570 DEBUG [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T08:21:22,650 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/table/cde2ec912f8b45d195a90bc74d67dca7 2024-11-25T08:21:22,657 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/info/8947e14d867e4f949d53c7d1abac4da9 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/info/8947e14d867e4f949d53c7d1abac4da9 2024-11-25T08:21:22,662 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/info/8947e14d867e4f949d53c7d1abac4da9, entries=69, sequenceid=59, filesize=13.8 K 2024-11-25T08:21:22,663 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/ns/39886e7e630949a48f456817ceac58db as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/ns/39886e7e630949a48f456817ceac58db 2024-11-25T08:21:22,668 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/ns/39886e7e630949a48f456817ceac58db, entries=7, sequenceid=59, filesize=5.2 K 2024-11-25T08:21:22,669 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/.tmp/table/cde2ec912f8b45d195a90bc74d67dca7 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/table/cde2ec912f8b45d195a90bc74d67dca7 2024-11-25T08:21:22,674 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/table/cde2ec912f8b45d195a90bc74d67dca7, entries=13, sequenceid=59, filesize=5.7 K 2024-11-25T08:21:22,675 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1307ms, sequenceid=59, compaction requested=false 2024-11-25T08:21:22,679 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-11-25T08:21:22,680 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-25T08:21:22,680 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T08:21:22,680 INFO [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T08:21:22,680 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732522881368Running coprocessor pre-close hooks at 1732522881368Disabling compacts and flushes for region at 1732522881368Disabling writes for close at 1732522881368Obtaining lock to block concurrent updates at 1732522881368Preparing flush snapshotting stores in 1588230740 at 1732522881368Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1732522881369 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732522881369Flushing 1588230740/info: creating writer at 1732522881369Flushing 1588230740/info: appending metadata at 1732522881392 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732522881392Flushing 1588230740/ns: creating writer at 1732522881803 (+411 ms)Flushing 1588230740/ns: appending metadata at 1732522881818 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732522881818Flushing 1588230740/table: creating writer at 1732522882230 (+412 ms)Flushing 1588230740/table: appending metadata at 1732522882244 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732522882245 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@518888e9: reopening flushed file at 1732522882656 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5676533e: reopening flushed file at 1732522882662 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34202161: reopening flushed file at 1732522882668 (+6 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1307ms, sequenceid=59, compaction requested=false at 1732522882675 (+7 ms)Writing region close event to WAL at 1732522882676 (+1 ms)Running coprocessor post-close hooks at 1732522882680 (+4 ms)Closed at 1732522882680 2024-11-25T08:21:22,680 DEBUG [RS_CLOSE_META-regionserver/5eb3d201e8c9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T08:21:22,770 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(976): stopping server 5eb3d201e8c9,43089,1732522777179; all regions closed. 2024-11-25T08:21:22,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741954_1130 (size=3260) 2024-11-25T08:21:22,774 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.meta.1732522870156.meta not finished, retry = 0 2024-11-25T08:21:22,879 DEBUG [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs 2024-11-25T08:21:22,879 INFO [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.meta:.meta(num 1732522870156) 2024-11-25T08:21:22,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741955_1131 (size=5275) 2024-11-25T08:21:22,881 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1.1732522870174 not finished, retry = 0 2024-11-25T08:21:22,985 DEBUG [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs 2024-11-25T08:21:22,985 INFO [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-1:(num 1732522870174) 2024-11-25T08:21:22,987 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/WALs/5eb3d201e8c9,43089,1732522777179/5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0.1732522870191 not finished, retry = 0 2024-11-25T08:21:22,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741956_1132 (size=1037) 2024-11-25T08:21:23,076 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-25T08:21:23,091 DEBUG [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/oldWALs 2024-11-25T08:21:23,091 INFO [RS:0;5eb3d201e8c9:43089 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5eb3d201e8c9%2C43089%2C1732522777179.5eb3d201e8c9%2C43089%2C1732522777179.regiongroup-0:(num 1732522870191) 2024-11-25T08:21:23,091 DEBUG [RS:0;5eb3d201e8c9:43089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:23,091 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T08:21:23,091 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T08:21:23,091 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.ChoreService(370): Chore service for: regionserver/5eb3d201e8c9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T08:21:23,091 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T08:21:23,092 INFO [regionserver/5eb3d201e8c9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T08:21:23,092 INFO [RS:0;5eb3d201e8c9:43089 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43089 2024-11-25T08:21:23,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/5eb3d201e8c9,43089,1732522777179 2024-11-25T08:21:23,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-25T08:21:23,099 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T08:21:23,102 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5eb3d201e8c9,43089,1732522777179] 2024-11-25T08:21:23,103 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/5eb3d201e8c9,43089,1732522777179 already deleted, retry=false 2024-11-25T08:21:23,103 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5eb3d201e8c9,43089,1732522777179 expired; onlineServers=0 2024-11-25T08:21:23,103 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5eb3d201e8c9,44851,1732522775045' ***** 2024-11-25T08:21:23,103 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T08:21:23,104 INFO [M:0;5eb3d201e8c9:44851 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-25T08:21:23,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-25T08:21:23,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-25T08:21:23,112 INFO [M:0;5eb3d201e8c9:44851 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@71a9493b{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-25T08:21:23,113 INFO [M:0;5eb3d201e8c9:44851 {}] server.AbstractConnector(383): Stopped ServerConnector@7e3adfd7{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-25T08:21:23,113 INFO [M:0;5eb3d201e8c9:44851 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T08:21:23,113 INFO [M:0;5eb3d201e8c9:44851 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6936b07c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-25T08:21:23,113 INFO [M:0;5eb3d201e8c9:44851 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@38ae1acb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,STOPPED} 2024-11-25T08:21:23,113 INFO [M:0;5eb3d201e8c9:44851 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T08:21:23,114 INFO [M:0;5eb3d201e8c9:44851 {}] master.HMaster(1770): Stopping master jetty server 2024-11-25T08:21:23,114 INFO [M:0;5eb3d201e8c9:44851 {}] server.AbstractConnector(383): Stopped ServerConnector@3007dbc3{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-25T08:21:23,115 INFO [M:0;5eb3d201e8c9:44851 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T08:21:23,115 DEBUG [M:0;5eb3d201e8c9:44851 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T08:21:23,115 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T08:21:23,115 INFO [M:0;5eb3d201e8c9:44851 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-25T08:21:23,115 INFO [M:0;5eb3d201e8c9:44851 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;5eb3d201e8c9:44851. 2024-11-25T08:21:23,115 DEBUG [M:0;5eb3d201e8c9:44851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T08:21:23,115 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.small.0-1732522781221 {}] cleaner.HFileCleaner(306): Exit Thread[master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.small.0-1732522781221,5,FailOnTimeoutGroup] 2024-11-25T08:21:23,115 DEBUG [master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.large.0-1732522781220 {}] cleaner.HFileCleaner(306): Exit Thread[master/5eb3d201e8c9:0:becomeActiveMaster-HFileCleaner.large.0-1732522781220,5,FailOnTimeoutGroup] 2024-11-25T08:21:23,115 DEBUG [M:0;5eb3d201e8c9:44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:23,116 DEBUG [M:0;5eb3d201e8c9:44851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T08:21:23,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T08:21:23,116 DEBUG [M:0;5eb3d201e8c9:44851 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T08:21:23,116 INFO [M:0;5eb3d201e8c9:44851 {}] hbase.ChoreService(370): Chore service for: master/5eb3d201e8c9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T08:21:23,116 INFO [M:0;5eb3d201e8c9:44851 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T08:21:23,116 DEBUG [M:0;5eb3d201e8c9:44851 {}] master.HMaster(1795): Stopping service threads 2024-11-25T08:21:23,116 INFO [M:0;5eb3d201e8c9:44851 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T08:21:23,117 INFO [M:0;5eb3d201e8c9:44851 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T08:21:23,117 INFO [M:0;5eb3d201e8c9:44851 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T08:21:23,117 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T08:21:23,118 DEBUG [M:0;5eb3d201e8c9:44851 {}] zookeeper.ZKUtil(347): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-25T08:21:23,118 WARN [M:0;5eb3d201e8c9:44851 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T08:21:23,119 INFO [M:0;5eb3d201e8c9:44851 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/.lastflushedseqids 2024-11-25T08:21:23,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741981_1157 (size=495) 2024-11-25T08:21:23,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T08:21:23,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43089-0x1014e35793c0001, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T08:21:23,203 INFO [RS:0;5eb3d201e8c9:43089 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T08:21:23,203 INFO [RS:0;5eb3d201e8c9:43089 {}] regionserver.HRegionServer(1031): Exiting; stopping=5eb3d201e8c9,43089,1732522777179; zookeeper connection closed. 2024-11-25T08:21:23,203 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6560a8e7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6560a8e7 2024-11-25T08:21:23,204 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T08:21:23,536 INFO [M:0;5eb3d201e8c9:44851 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T08:21:23,536 INFO [M:0;5eb3d201e8c9:44851 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T08:21:23,536 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T08:21:23,550 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T08:21:23,550 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T08:21:23,550 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T08:21:23,550 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T08:21:23,550 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=318.97 KB heapSize=374.92 KB 2024-11-25T08:21:23,567 DEBUG [M:0;5eb3d201e8c9:44851 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/98b50e31fbb246de8bd299c3cb913bf5 is 82, key is hbase:meta,,1/info:regioninfo/1732522782564/Put/seqid=0 2024-11-25T08:21:23,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741982_1158 (size=5672) 2024-11-25T08:21:23,971 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/98b50e31fbb246de8bd299c3cb913bf5 2024-11-25T08:21:23,993 DEBUG [M:0;5eb3d201e8c9:44851 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e528b6c694e0441d8acdd7db52b1f744 is 2183, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1732522831446/Put/seqid=0 2024-11-25T08:21:23,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741983_1159 (size=26496) 2024-11-25T08:21:24,398 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.41 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e528b6c694e0441d8acdd7db52b1f744 2024-11-25T08:21:24,403 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e528b6c694e0441d8acdd7db52b1f744 2024-11-25T08:21:24,418 DEBUG [M:0;5eb3d201e8c9:44851 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/249e554407f44c9da45151c01ca8484d is 69, key is 5eb3d201e8c9,43089,1732522777179/rs:state/1732522781235/Put/seqid=0 2024-11-25T08:21:24,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741984_1160 (size=5156) 2024-11-25T08:21:24,823 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/249e554407f44c9da45151c01ca8484d 2024-11-25T08:21:24,829 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/98b50e31fbb246de8bd299c3cb913bf5 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/98b50e31fbb246de8bd299c3cb913bf5 2024-11-25T08:21:24,834 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/98b50e31fbb246de8bd299c3cb913bf5, entries=8, sequenceid=667, filesize=5.5 K 2024-11-25T08:21:24,835 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e528b6c694e0441d8acdd7db52b1f744 as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e528b6c694e0441d8acdd7db52b1f744 2024-11-25T08:21:24,840 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e528b6c694e0441d8acdd7db52b1f744 2024-11-25T08:21:24,840 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e528b6c694e0441d8acdd7db52b1f744, entries=69, sequenceid=667, filesize=25.9 K 2024-11-25T08:21:24,841 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/249e554407f44c9da45151c01ca8484d as hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/249e554407f44c9da45151c01ca8484d 2024-11-25T08:21:24,847 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/249e554407f44c9da45151c01ca8484d, entries=1, sequenceid=667, filesize=5.0 K 2024-11-25T08:21:24,848 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(3140): Finished flush of dataSize ~318.97 KB/326621, heapSize ~374.63 KB/383616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1298ms, sequenceid=667, compaction requested=false 2024-11-25T08:21:24,850 INFO [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T08:21:24,850 DEBUG [M:0;5eb3d201e8c9:44851 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732522883536Disabling compacts and flushes for region at 1732522883536Disabling writes for close at 1732522883550 (+14 ms)Obtaining lock to block concurrent updates at 1732522883550Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732522883550Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326621, getHeapSize=383856, getOffHeapSize=0, getCellsCount=746 at 1732522883551 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732522883551Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732522883551Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732522883566 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732522883566Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732522883977 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732522883992 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732522883992Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732522884403 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732522884418 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732522884418Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76ec3d5f: reopening flushed file at 1732522884828 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b20b899: reopening flushed file at 1732522884834 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a946ee2: reopening flushed file at 1732522884840 (+6 ms)Finished flush of dataSize ~318.97 KB/326621, heapSize ~374.63 KB/383616, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1298ms, sequenceid=667, compaction requested=false at 1732522884848 (+8 ms)Writing region close event to WAL at 1732522884849 (+1 ms)Closed at 1732522884849 2024-11-25T08:21:24,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43131 is added to blk_1073741830_1006 (size=370711) 2024-11-25T08:21:24,853 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0484a00f-2eef-f94a-c280-331c69da9927/MasterData/WALs/5eb3d201e8c9,44851,1732522775045/5eb3d201e8c9%2C44851%2C1732522775045.5eb3d201e8c9%2C44851%2C1732522775045.regiongroup-0.1732522779499 not finished, retry = 0 2024-11-25T08:21:24,954 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T08:21:24,954 INFO [M:0;5eb3d201e8c9:44851 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T08:21:24,954 INFO [M:0;5eb3d201e8c9:44851 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-25T08:21:24,955 INFO [M:0;5eb3d201e8c9:44851 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44851 2024-11-25T08:21:24,955 INFO [M:0;5eb3d201e8c9:44851 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T08:21:25,057 INFO [M:0;5eb3d201e8c9:44851 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T08:21:25,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T08:21:25,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44851-0x1014e35793c0000, quorum=127.0.0.1:63891, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T08:21:25,065 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6acf7532{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T08:21:25,067 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74250dde{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T08:21:25,067 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T08:21:25,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c1aa6bd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T08:21:25,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41edcc56{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,STOPPED} 2024-11-25T08:21:25,070 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T08:21:25,070 WARN [BP-814101347-172.17.0.2-1732522768861 heartbeating to localhost/127.0.0.1:42243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T08:21:25,070 WARN [BP-814101347-172.17.0.2-1732522768861 heartbeating to localhost/127.0.0.1:42243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-814101347-172.17.0.2-1732522768861 (Datanode Uuid ccdb6c77-1653-4576-9bc7-57cbc5a32064) service to localhost/127.0.0.1:42243 2024-11-25T08:21:25,070 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T08:21:25,072 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/data/data1/current/BP-814101347-172.17.0.2-1732522768861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T08:21:25,072 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/cluster_cbe8eec6-ed56-7099-2c40-652023936def/data/data2/current/BP-814101347-172.17.0.2-1732522768861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T08:21:25,073 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T08:21:25,080 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ee33b9d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T08:21:25,080 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a655ef8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T08:21:25,080 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T08:21:25,081 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e61513{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T08:21:25,081 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12f83a6b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,STOPPED} 2024-11-25T08:21:25,089 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T08:21:25,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T08:21:25,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-25T08:21:25,150 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4611faf6{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-25T08:21:25,150 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f98650{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T08:21:25,151 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T08:21:25,151 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fdfb8bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-25T08:21:25,151 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f552527{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,STOPPED} 2024-11-25T08:21:25,160 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40a758a4{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-25T08:21:25,161 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4a6d3ac6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T08:21:25,161 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T08:21:25,161 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@27083e15{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-25T08:21:25,161 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48e39004{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,STOPPED} 2024-11-25T08:21:25,167 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-25T08:21:25,167 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3bcbb650{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-25T08:21:25,167 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@20db1e36{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T08:21:25,167 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T08:21:25,168 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@234d7a74{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-25T08:21:25,168 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fa5f4d9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,STOPPED} 2024-11-25T08:21:25,171 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-25T08:21:25,176 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-25T08:21:25,176 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-25T08:21:25,177 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-25T08:21:25,181 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@14baf7c7{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-25T08:21:25,182 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54ee21da{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T08:21:25,182 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T08:21:25,182 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fa2e322{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-25T08:21:25,182 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b1ead7d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/6e750cc4-c5eb-50c5-08de-d1aeb3732828/hadoop.log.dir/,STOPPED} 2024-11-25T08:21:25,184 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-25T08:21:25,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped