2024-11-28 02:15:09,431 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-28 02:15:09,442 main DEBUG Took 0.009770 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 02:15:09,443 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 02:15:09,443 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 02:15:09,444 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 02:15:09,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,456 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 02:15:09,511 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,513 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,514 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,515 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,516 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,518 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,519 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,520 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,521 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,523 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,524 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,525 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,525 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,526 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,527 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 02:15:09,528 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,528 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 02:15:09,530 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 02:15:09,532 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 02:15:09,535 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 02:15:09,535 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 02:15:09,537 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 02:15:09,537 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 02:15:09,550 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 02:15:09,554 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 02:15:09,557 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 02:15:09,557 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 02:15:09,558 main DEBUG createAppenders(={Console}) 2024-11-28 02:15:09,559 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-28 02:15:09,559 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-28 02:15:09,559 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-28 02:15:09,560 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 02:15:09,560 main DEBUG OutputStream closed 2024-11-28 02:15:09,560 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 02:15:09,561 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 02:15:09,561 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-28 02:15:09,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 02:15:09,654 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 02:15:09,656 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 02:15:09,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 02:15:09,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 02:15:09,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 02:15:09,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 02:15:09,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 02:15:09,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 02:15:09,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 02:15:09,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 02:15:09,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 02:15:09,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 02:15:09,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 02:15:09,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 02:15:09,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 02:15:09,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 02:15:09,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 02:15:09,665 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 02:15:09,665 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-28 02:15:09,665 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 02:15:09,666 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-28T02:15:09,713 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-28 02:15:09,716 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 02:15:09,717 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T02:15:10,059 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098 2024-11-28T02:15:10,064 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-28T02:15:10,067 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:15:10,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T02:15:10,088 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd, deleteOnExit=true 2024-11-28T02:15:10,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T02:15:10,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/test.cache.data in system properties and HBase conf 2024-11-28T02:15:10,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T02:15:10,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir in system properties and HBase conf 2024-11-28T02:15:10,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T02:15:10,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T02:15:10,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T02:15:10,210 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T02:15:10,311 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T02:15:10,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T02:15:10,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T02:15:10,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T02:15:10,317 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T02:15:10,317 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T02:15:10,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T02:15:10,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T02:15:10,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T02:15:10,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T02:15:10,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/nfs.dump.dir in system properties and HBase conf 2024-11-28T02:15:10,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/java.io.tmpdir in system properties and HBase conf 2024-11-28T02:15:10,321 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T02:15:10,322 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T02:15:10,322 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T02:15:11,183 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T02:15:11,308 INFO [Time-limited test {}] log.Log(170): Logging initialized @2658ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T02:15:11,405 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:11,480 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T02:15:11,511 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:15:11,511 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:15:11,513 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T02:15:11,535 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:11,538 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f045f3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:15:11,540 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b22dca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T02:15:11,768 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@38e78c7b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/java.io.tmpdir/jetty-localhost-34885-hadoop-hdfs-3_4_1-tests_jar-_-any-12605469811598218172/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T02:15:11,780 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c336369{HTTP/1.1, (http/1.1)}{localhost:34885} 2024-11-28T02:15:11,780 INFO [Time-limited test {}] server.Server(415): Started @3131ms 2024-11-28T02:15:12,185 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:12,192 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T02:15:12,193 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:15:12,193 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:15:12,193 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T02:15:12,194 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:15:12,195 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T02:15:12,326 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@cb44ad4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/java.io.tmpdir/jetty-localhost-46081-hadoop-hdfs-3_4_1-tests_jar-_-any-769985205598241726/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T02:15:12,327 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:46081} 2024-11-28T02:15:12,327 INFO [Time-limited test {}] server.Server(415): Started @3678ms 2024-11-28T02:15:12,395 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T02:15:12,995 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/data/data1/current/BP-903783156-172.17.0.2-1732760110915/current, will proceed with Du for space computation calculation, 2024-11-28T02:15:12,995 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/data/data2/current/BP-903783156-172.17.0.2-1732760110915/current, will proceed with Du for space computation calculation, 2024-11-28T02:15:13,042 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T02:15:13,103 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d0fb6a348827ab9 with lease ID 0x54624e2349d0bad4: Processing first storage report for DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8 from datanode DatanodeRegistration(127.0.0.1:37989, datanodeUuid=96d72808-c4db-44d7-90dc-7b59517a8479, infoPort=41009, infoSecurePort=0, ipcPort=35159, storageInfo=lv=-57;cid=testClusterID;nsid=1674796324;c=1732760110915) 2024-11-28T02:15:13,104 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d0fb6a348827ab9 with lease ID 0x54624e2349d0bad4: from storage DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8 node DatanodeRegistration(127.0.0.1:37989, datanodeUuid=96d72808-c4db-44d7-90dc-7b59517a8479, infoPort=41009, infoSecurePort=0, ipcPort=35159, storageInfo=lv=-57;cid=testClusterID;nsid=1674796324;c=1732760110915), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-28T02:15:13,105 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d0fb6a348827ab9 with lease ID 0x54624e2349d0bad4: Processing first storage report for DS-02766c8e-a56a-4ee8-aff0-79955a765baa from datanode DatanodeRegistration(127.0.0.1:37989, datanodeUuid=96d72808-c4db-44d7-90dc-7b59517a8479, infoPort=41009, infoSecurePort=0, ipcPort=35159, storageInfo=lv=-57;cid=testClusterID;nsid=1674796324;c=1732760110915) 2024-11-28T02:15:13,106 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d0fb6a348827ab9 with lease ID 0x54624e2349d0bad4: from storage DS-02766c8e-a56a-4ee8-aff0-79955a765baa node DatanodeRegistration(127.0.0.1:37989, datanodeUuid=96d72808-c4db-44d7-90dc-7b59517a8479, infoPort=41009, infoSecurePort=0, ipcPort=35159, storageInfo=lv=-57;cid=testClusterID;nsid=1674796324;c=1732760110915), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T02:15:13,124 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098 2024-11-28T02:15:13,234 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/zookeeper_0, clientPort=50908, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T02:15:13,248 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50908 2024-11-28T02:15:13,259 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:15:13,261 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:15:13,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741825_1001 (size=7) 2024-11-28T02:15:13,914 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 with version=8 2024-11-28T02:15:13,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/hbase-staging 2024-11-28T02:15:13,931 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-28T02:15:13,932 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-28T02:15:13,932 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-28T02:15:13,932 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-28T02:15:14,027 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T02:15:14,294 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d9827d9b919f:0 server-side Connection retries=45 2024-11-28T02:15:14,305 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T02:15:14,306 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T02:15:14,310 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T02:15:14,311 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T02:15:14,311 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T02:15:14,447 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T02:15:14,513 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T02:15:14,525 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T02:15:14,531 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T02:15:14,560 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28143 (auto-detected) 2024-11-28T02:15:14,561 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T02:15:14,580 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43413 2024-11-28T02:15:14,604 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43413 connecting to ZooKeeper ensemble=127.0.0.1:50908 2024-11-28T02:15:14,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:434130x0, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T02:15:14,646 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43413-0x1015c5afacc0000 connected 2024-11-28T02:15:14,675 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:15:14,677 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:15:14,687 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-28T02:15:14,709 INFO [Time-limited test {}] log.Log(170): Logging initialized @6060ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T02:15:14,756 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-28T02:15:14,756 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-28T02:15:14,757 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-28T02:15:14,758 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-28T02:15:14,758 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-28T02:15:14,760 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-28T02:15:14,761 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-28T02:15:14,761 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-28T02:15:14,763 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-28T02:15:14,833 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 46053 2024-11-28T02:15:14,834 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-28T02:15:14,855 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-28T02:15:14,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7c1597dd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:15:14,858 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-28T02:15:14,858 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@65507143{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-28T02:15:14,907 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-28T02:15:14,914 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:15:14,914 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:15:14,915 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T02:15:14,921 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-28T02:15:15,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5c685ba8{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-28T02:15:15,486 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10f9ba48{HTTP/1.1, (http/1.1)}{0.0.0.0:46053} 2024-11-28T02:15:15,487 INFO [Time-limited test {}] server.Server(415): Started @6838ms 2024-11-28T02:15:15,490 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96, hbase.cluster.distributed=false 2024-11-28T02:15:15,516 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-28T02:15:15,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43413 2024-11-28T02:15:15,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43413 2024-11-28T02:15:15,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43413 2024-11-28T02:15:15,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43413 2024-11-28T02:15:15,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43413 2024-11-28T02:15:15,638 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9827d9b919f:0 server-side Connection retries=45 2024-11-28T02:15:15,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T02:15:15,640 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T02:15:15,640 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T02:15:15,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T02:15:15,641 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T02:15:15,644 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T02:15:15,646 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T02:15:15,647 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42127 2024-11-28T02:15:15,649 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42127 connecting to ZooKeeper ensemble=127.0.0.1:50908 2024-11-28T02:15:15,650 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:15:15,653 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:15:15,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421270x0, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T02:15:15,660 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42127-0x1015c5afacc0001 connected 2024-11-28T02:15:15,661 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-28T02:15:15,663 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-28T02:15:15,663 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-28T02:15:15,663 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-28T02:15:15,663 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-28T02:15:15,664 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-28T02:15:15,665 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-28T02:15:15,665 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-28T02:15:15,665 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-28T02:15:15,665 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-28T02:15:15,667 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 38595 2024-11-28T02:15:15,667 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-28T02:15:15,670 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-28T02:15:15,670 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3e9615b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:15:15,670 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-28T02:15:15,671 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2df3a35c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-28T02:15:15,688 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-28T02:15:15,689 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:15:15,689 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:15:15,690 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T02:15:15,691 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-28T02:15:15,695 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@9984275{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-28T02:15:15,696 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@445a8045{HTTP/1.1, (http/1.1)}{0.0.0.0:38595} 2024-11-28T02:15:15,696 INFO [Time-limited test {}] server.Server(415): Started @7047ms 2024-11-28T02:15:15,701 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T02:15:15,710 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T02:15:15,713 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-28T02:15:15,718 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-28T02:15:15,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42127 2024-11-28T02:15:15,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42127 2024-11-28T02:15:15,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42127 2024-11-28T02:15:15,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42127 2024-11-28T02:15:15,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42127 2024-11-28T02:15:15,738 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d9827d9b919f:43413 2024-11-28T02:15:15,741 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-28T02:15:15,745 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@106565e3{HTTP/1.1, (http/1.1)}{0.0.0.0:37651} 2024-11-28T02:15:15,745 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] server.Server(415): Started @7096ms 2024-11-28T02:15:15,746 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/d9827d9b919f,43413,1732760114080 2024-11-28T02:15:15,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-28T02:15:15,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-28T02:15:15,755 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/d9827d9b919f,43413,1732760114080 2024-11-28T02:15:15,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-28T02:15:15,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:15,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:15,779 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-28T02:15:15,780 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/d9827d9b919f,43413,1732760114080 from backup master directory 2024-11-28T02:15:15,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/d9827d9b919f,43413,1732760114080 2024-11-28T02:15:15,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-28T02:15:15,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-28T02:15:15,784 WARN [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T02:15:15,784 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d9827d9b919f,43413,1732760114080 2024-11-28T02:15:15,787 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T02:15:15,788 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T02:15:15,845 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/hbase.id] with ID: e93b2abd-558d-40dd-8d9f-9c3f791c7807 2024-11-28T02:15:15,845 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.tmp/hbase.id 2024-11-28T02:15:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741826_1002 (size=42) 2024-11-28T02:15:16,257 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.tmp/hbase.id]:[hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/hbase.id] 2024-11-28T02:15:16,299 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T02:15:16,304 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T02:15:16,324 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-28T02:15:16,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:16,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:16,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741827_1003 (size=196) 2024-11-28T02:15:16,759 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:16,761 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T02:15:16,768 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-28T02:15:16,770 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-28T02:15:16,782 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:15:16,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741828_1004 (size=1189) 2024-11-28T02:15:16,836 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store 2024-11-28T02:15:16,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741829_1005 (size=34) 2024-11-28T02:15:17,260 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T02:15:17,265 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:17,266 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T02:15:17,267 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:15:17,267 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:15:17,269 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T02:15:17,269 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:15:17,270 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:15:17,271 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732760117266Disabling compacts and flushes for region at 1732760117266Disabling writes for close at 1732760117269 (+3 ms)Writing region close event to WAL at 1732760117270 (+1 ms)Closed at 1732760117270 2024-11-28T02:15:17,274 WARN [master/d9827d9b919f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/.initializing 2024-11-28T02:15:17,274 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/WALs/d9827d9b919f,43413,1732760114080 2024-11-28T02:15:17,279 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T02:15:17,285 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9827d9b919f,43413,1732760114080.regiongroup-0 2024-11-28T02:15:17,300 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9827d9b919f%2C43413%2C1732760114080.d9827d9b919f%2C43413%2C1732760114080.regiongroup-0, suffix=, logDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/WALs/d9827d9b919f,43413,1732760114080, archiveDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/oldWALs, maxLogs=10 2024-11-28T02:15:17,324 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/WALs/d9827d9b919f,43413,1732760114080/d9827d9b919f%2C43413%2C1732760114080.d9827d9b919f%2C43413%2C1732760114080.regiongroup-0.1732760117304, exclude list is [], retry=0 2024-11-28T02:15:17,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:17,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T02:15:17,382 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/WALs/d9827d9b919f,43413,1732760114080/d9827d9b919f%2C43413%2C1732760114080.d9827d9b919f%2C43413%2C1732760114080.regiongroup-0.1732760117304 2024-11-28T02:15:17,383 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:17,383 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:17,384 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:17,387 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,388 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T02:15:17,464 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:17,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:17,467 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T02:15:17,470 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:17,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:17,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T02:15:17,474 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:17,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:17,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,478 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T02:15:17,478 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:17,479 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:17,479 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,482 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,484 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,489 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,490 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,493 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T02:15:17,497 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T02:15:17,502 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:15:17,503 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11945950080, jitterRate=0.11255329847335815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T02:15:17,511 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732760117405Initializing all the Stores at 1732760117407 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732760117407Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760117409 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760117409Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760117410 (+1 ms)Cleaning up temporary data from old regions at 1732760117490 (+80 ms)Region opened successfully at 1732760117510 (+20 ms) 2024-11-28T02:15:17,512 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T02:15:17,547 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b04e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=d9827d9b919f/172.17.0.2:0 2024-11-28T02:15:17,582 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T02:15:17,594 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T02:15:17,594 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T02:15:17,598 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T02:15:17,599 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T02:15:17,605 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-28T02:15:17,605 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T02:15:17,635 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T02:15:17,645 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-28T02:15:17,647 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-28T02:15:17,650 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T02:15:17,652 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-28T02:15:17,654 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-28T02:15:17,656 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T02:15:17,660 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-28T02:15:17,663 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-28T02:15:17,665 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-28T02:15:17,666 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-28T02:15:17,710 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T02:15:17,712 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-28T02:15:17,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-28T02:15:17,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-28T02:15:17,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:17,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:17,719 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d9827d9b919f,43413,1732760114080, sessionid=0x1015c5afacc0000, setting cluster-up flag (Was=false) 2024-11-28T02:15:17,726 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-28T02:15:17,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:17,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:17,746 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-28T02:15:17,747 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9827d9b919f,43413,1732760114080 2024-11-28T02:15:17,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:17,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:17,758 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-28T02:15:17,759 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9827d9b919f,43413,1732760114080 2024-11-28T02:15:17,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:17,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:17,770 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-28T02:15:17,772 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9827d9b919f,43413,1732760114080 2024-11-28T02:15:17,778 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T02:15:17,827 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(746): ClusterId : e93b2abd-558d-40dd-8d9f-9c3f791c7807 2024-11-28T02:15:17,828 INFO [RS:0;d9827d9b919f:42127 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-28T02:15:17,830 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T02:15:17,835 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T02:15:17,835 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-28T02:15:17,840 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-28T02:15:17,840 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T02:15:17,843 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T02:15:17,844 DEBUG [RS:0;d9827d9b919f:42127 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d4f88f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=d9827d9b919f/172.17.0.2:0 2024-11-28T02:15:17,861 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T02:15:17,865 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d9827d9b919f:42127 2024-11-28T02:15:17,868 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T02:15:17,869 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T02:15:17,869 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T02:15:17,871 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T02:15:17,871 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9827d9b919f,43413,1732760114080 with port=42127, startcode=1732760115598 2024-11-28T02:15:17,877 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T02:15:17,882 DEBUG [RS:0;d9827d9b919f:42127 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T02:15:17,882 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d9827d9b919f,43413,1732760114080 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T02:15:17,889 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d9827d9b919f:0, corePoolSize=5, maxPoolSize=5 2024-11-28T02:15:17,889 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d9827d9b919f:0, corePoolSize=5, maxPoolSize=5 2024-11-28T02:15:17,889 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d9827d9b919f:0, corePoolSize=5, maxPoolSize=5 2024-11-28T02:15:17,889 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d9827d9b919f:0, corePoolSize=5, maxPoolSize=5 2024-11-28T02:15:17,890 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d9827d9b919f:0, corePoolSize=10, maxPoolSize=10 2024-11-28T02:15:17,890 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:17,890 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d9827d9b919f:0, corePoolSize=2, maxPoolSize=2 2024-11-28T02:15:17,890 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:17,893 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732760147893 2024-11-28T02:15:17,895 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T02:15:17,895 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T02:15:17,895 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T02:15:17,896 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T02:15:17,900 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T02:15:17,901 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-28T02:15:17,901 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T02:15:17,901 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T02:15:17,901 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T02:15:17,903 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:17,903 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T02:15:17,903 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:17,909 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T02:15:17,910 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T02:15:17,910 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T02:15:17,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741831_1007 (size=1321) 2024-11-28T02:15:17,922 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50703, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T02:15:17,925 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9cfb2a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:17,925 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:17,928 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:17,929 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43413 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-28T02:15:17,932 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:17,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:17,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:17,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d8c895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:17,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:17,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:17,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:17,949 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:17,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d749e32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:17,953 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-28T02:15:17,955 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T02:15:17,956 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T02:15:17,958 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d9827d9b919f:0:becomeActiveMaster-HFileCleaner.large.0-1732760117957,5,FailOnTimeoutGroup] 2024-11-28T02:15:17,958 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d9827d9b919f:0:becomeActiveMaster-HFileCleaner.small.0-1732760117958,5,FailOnTimeoutGroup] 2024-11-28T02:15:17,958 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:17,959 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T02:15:17,960 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:17,960 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:17,960 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-28T02:15:17,960 WARN [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-28T02:15:18,062 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9827d9b919f,43413,1732760114080 with port=42127, startcode=1732760115598 2024-11-28T02:15:18,064 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43413 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9827d9b919f,42127,1732760115598 2024-11-28T02:15:18,066 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43413 {}] master.ServerManager(517): Registering regionserver=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:18,077 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:18,077 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38631 2024-11-28T02:15:18,077 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=46053 2024-11-28T02:15:18,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-28T02:15:18,085 DEBUG [RS:0;d9827d9b919f:42127 {}] zookeeper.ZKUtil(111): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rs/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:18,085 WARN [RS:0;d9827d9b919f:42127 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T02:15:18,085 INFO [RS:0;d9827d9b919f:42127 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-28T02:15:18,085 INFO [RS:0;d9827d9b919f:42127 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-28T02:15:18,085 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:18,087 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9827d9b919f,42127,1732760115598] 2024-11-28T02:15:18,111 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T02:15:18,125 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T02:15:18,131 INFO [RS:0;d9827d9b919f:42127 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T02:15:18,131 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,132 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T02:15:18,139 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T02:15:18,141 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,141 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,141 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,141 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,141 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,142 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,142 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9827d9b919f:0, corePoolSize=2, maxPoolSize=2 2024-11-28T02:15:18,142 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,142 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,142 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,142 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,142 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,142 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9827d9b919f:0, corePoolSize=1, maxPoolSize=1 2024-11-28T02:15:18,143 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0, corePoolSize=3, maxPoolSize=3 2024-11-28T02:15:18,143 DEBUG [RS:0;d9827d9b919f:42127 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9827d9b919f:0, corePoolSize=3, maxPoolSize=3 2024-11-28T02:15:18,144 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,144 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,144 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,144 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,144 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,144 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9827d9b919f,42127,1732760115598-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T02:15:18,162 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T02:15:18,164 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9827d9b919f,42127,1732760115598-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,164 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,164 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.Replication(171): d9827d9b919f,42127,1732760115598 started 2024-11-28T02:15:18,181 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:18,181 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1482): Serving as d9827d9b919f,42127,1732760115598, RpcServer on d9827d9b919f/172.17.0.2:42127, sessionid=0x1015c5afacc0001 2024-11-28T02:15:18,182 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T02:15:18,182 DEBUG [RS:0;d9827d9b919f:42127 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9827d9b919f,42127,1732760115598 2024-11-28T02:15:18,182 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9827d9b919f,42127,1732760115598' 2024-11-28T02:15:18,182 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-28T02:15:18,183 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-28T02:15:18,184 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T02:15:18,184 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-28T02:15:18,184 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9827d9b919f,42127,1732760115598' 2024-11-28T02:15:18,184 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:15:18,184 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:15:18,185 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-28T02:15:18,185 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-28T02:15:18,185 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T02:15:18,185 DEBUG [RS:0;d9827d9b919f:42127 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9827d9b919f,42127,1732760115598 2024-11-28T02:15:18,185 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9827d9b919f,42127,1732760115598' 2024-11-28T02:15:18,185 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-28T02:15:18,186 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-28T02:15:18,187 DEBUG [RS:0;d9827d9b919f:42127 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T02:15:18,187 INFO [RS:0;d9827d9b919f:42127 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T02:15:18,187 INFO [RS:0;d9827d9b919f:42127 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T02:15:18,292 INFO [RS:0;d9827d9b919f:42127 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T02:15:18,292 INFO [RS:0;d9827d9b919f:42127 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9827d9b919f,42127,1732760115598.regiongroup-0 2024-11-28T02:15:18,296 INFO [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0, suffix=, logDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598, archiveDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs, maxLogs=32 2024-11-28T02:15:18,312 DEBUG [RS:0;d9827d9b919f:42127 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760118299, exclude list is [], retry=0 2024-11-28T02:15:18,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:18,320 INFO [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760118299 2024-11-28T02:15:18,321 DEBUG [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:18,323 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T02:15:18,324 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:18,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741833_1009 (size=32) 2024-11-28T02:15:18,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:18,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T02:15:18,741 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T02:15:18,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:18,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:18,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T02:15:18,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T02:15:18,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:18,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:18,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T02:15:18,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T02:15:18,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:18,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:18,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T02:15:18,751 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T02:15:18,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:18,752 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:18,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T02:15:18,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740 2024-11-28T02:15:18,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740 2024-11-28T02:15:18,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T02:15:18,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T02:15:18,758 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T02:15:18,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T02:15:18,763 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:15:18,764 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11075698080, jitterRate=0.03150476515293121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T02:15:18,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732760118736Initializing all the Stores at 1732760118738 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732760118739 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732760118739Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760118739Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732760118739Cleaning up temporary data from old regions at 1732760118757 (+18 ms)Region opened successfully at 1732760118766 (+9 ms) 2024-11-28T02:15:18,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T02:15:18,767 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T02:15:18,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T02:15:18,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T02:15:18,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T02:15:18,768 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T02:15:18,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732760118766Disabling compacts and flushes for region at 1732760118766Disabling writes for close at 1732760118767 (+1 ms)Writing region close event to WAL at 1732760118768 (+1 ms)Closed at 1732760118768 2024-11-28T02:15:18,772 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T02:15:18,772 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T02:15:18,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T02:15:18,786 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T02:15:18,789 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T02:15:18,942 DEBUG [d9827d9b919f:43413 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T02:15:18,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:18,965 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9827d9b919f,42127,1732760115598, state=OPENING 2024-11-28T02:15:18,970 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T02:15:18,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:18,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:15:18,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-28T02:15:18,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-28T02:15:18,973 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T02:15:18,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:19,150 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T02:15:19,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48897, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T02:15:19,163 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T02:15:19,163 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-28T02:15:19,164 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-28T02:15:19,164 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T02:15:19,164 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T02:15:19,167 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9827d9b919f%2C42127%2C1732760115598.meta, suffix=.meta, logDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598, archiveDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs, maxLogs=32 2024-11-28T02:15:19,183 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760119169.meta, exclude list is [], retry=0 2024-11-28T02:15:19,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:19,190 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760119169.meta 2024-11-28T02:15:19,190 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:19,190 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:19,194 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:15:19,195 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T02:15:19,198 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T02:15:19,198 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T02:15:19,202 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T02:15:19,202 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:19,202 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T02:15:19,203 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T02:15:19,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T02:15:19,207 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T02:15:19,208 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:19,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:19,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T02:15:19,211 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T02:15:19,211 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:19,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:19,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T02:15:19,213 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T02:15:19,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:19,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:19,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T02:15:19,216 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T02:15:19,216 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:19,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T02:15:19,217 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T02:15:19,219 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740 2024-11-28T02:15:19,221 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740 2024-11-28T02:15:19,224 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T02:15:19,224 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T02:15:19,225 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T02:15:19,227 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T02:15:19,229 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9622013920, jitterRate=-0.10388012230396271}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T02:15:19,229 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T02:15:19,231 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732760119203Writing region info on filesystem at 1732760119203Initializing all the Stores at 1732760119205 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732760119205Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732760119205Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760119205Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732760119205Cleaning up temporary data from old regions at 1732760119224 (+19 ms)Running coprocessor post-open hooks at 1732760119229 (+5 ms)Region opened successfully at 1732760119231 (+2 ms) 2024-11-28T02:15:19,238 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732760119141 2024-11-28T02:15:19,250 DEBUG [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T02:15:19,250 INFO [RS_OPEN_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T02:15:19,252 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:19,254 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9827d9b919f,42127,1732760115598, state=OPEN 2024-11-28T02:15:19,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-28T02:15:19,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-28T02:15:19,258 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-28T02:15:19,258 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-28T02:15:19,258 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:19,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T02:15:19,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d9827d9b919f,42127,1732760115598 in 283 msec 2024-11-28T02:15:19,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T02:15:19,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 486 msec 2024-11-28T02:15:19,270 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T02:15:19,271 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T02:15:19,291 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:19,292 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:19,312 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:19,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47835, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:19,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5270 sec 2024-11-28T02:15:19,338 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732760119337, completionTime=-1 2024-11-28T02:15:19,341 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T02:15:19,341 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T02:15:19,370 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T02:15:19,371 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732760179371 2024-11-28T02:15:19,371 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732760239371 2024-11-28T02:15:19,371 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-28T02:15:19,373 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9827d9b919f,43413,1732760114080-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:19,373 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9827d9b919f,43413,1732760114080-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:19,373 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9827d9b919f,43413,1732760114080-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:19,374 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d9827d9b919f:43413, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:19,375 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:19,375 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:19,380 DEBUG [master/d9827d9b919f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T02:15:19,404 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.619sec 2024-11-28T02:15:19,405 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T02:15:19,406 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T02:15:19,407 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T02:15:19,408 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T02:15:19,408 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T02:15:19,409 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9827d9b919f,43413,1732760114080-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T02:15:19,409 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9827d9b919f,43413,1732760114080-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T02:15:19,418 DEBUG [master/d9827d9b919f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T02:15:19,418 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T02:15:19,419 INFO [master/d9827d9b919f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9827d9b919f,43413,1732760114080-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T02:15:19,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cc25be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:19,432 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T02:15:19,432 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T02:15:19,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:19,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:19,442 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:19,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5273da3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:19,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:19,447 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:19,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195abe33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:19,449 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:19,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:19,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d9827d9b919f,43413,1732760114080 2024-11-28T02:15:19,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-28T02:15:19,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/test.cache.data in system properties and HBase conf 2024-11-28T02:15:19,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T02:15:19,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir in system properties and HBase conf 2024-11-28T02:15:19,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T02:15:19,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T02:15:19,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/nfs.dump.dir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/java.io.tmpdir in system properties and HBase conf 2024-11-28T02:15:19,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T02:15:19,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T02:15:19,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T02:15:19,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741835_1011 (size=592039) 2024-11-28T02:15:19,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741836_1012 (size=1663647) 2024-11-28T02:15:21,494 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:21,625 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:21,785 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T02:15:21,788 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:15:21,788 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:15:21,788 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T02:15:21,789 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@171eacd3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:15:21,789 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-28T02:15:21,789 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a60e5b2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-28T02:15:21,790 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T02:15:21,804 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:15:21,805 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:15:21,805 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T02:15:21,807 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:21,815 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ccdffea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:15:21,816 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31795375{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-28T02:15:21,937 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2024-11-28T02:15:21,937 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2024-11-28T02:15:21,937 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-28T02:15:21,939 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-28T02:15:21,979 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:22,068 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:22,270 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:22,289 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bdc42df{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/java.io.tmpdir/jetty-localhost-40453-hadoop-yarn-common-3_4_1_jar-_-any-12758552604943728640/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-28T02:15:22,289 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@31ab3c62{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/java.io.tmpdir/jetty-localhost-41031-hadoop-yarn-common-3_4_1_jar-_-any-13441853422610004394/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-28T02:15:22,290 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@567287a2{HTTP/1.1, (http/1.1)}{localhost:40453} 2024-11-28T02:15:22,290 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@15a81a3d{HTTP/1.1, (http/1.1)}{localhost:41031} 2024-11-28T02:15:22,290 INFO [Time-limited test {}] server.Server(415): Started @13641ms 2024-11-28T02:15:22,290 INFO [Thread-251 {}] server.Server(415): Started @13641ms 2024-11-28T02:15:22,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741837_1013 (size=5) 2024-11-28T02:15:23,624 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-28T02:15:23,628 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:23,661 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-28T02:15:23,662 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T02:15:23,665 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:15:23,665 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:15:23,666 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T02:15:23,666 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:23,667 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9e11c64{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:15:23,668 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26a51899{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-28T02:15:23,739 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-28T02:15:23,739 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-28T02:15:23,739 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-28T02:15:23,739 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-28T02:15:23,747 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:23,753 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:23,889 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:23,902 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55049bf1{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/java.io.tmpdir/jetty-localhost-37145-hadoop-yarn-common-3_4_1_jar-_-any-2709838474775825250/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-28T02:15:23,902 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@534a2f07{HTTP/1.1, (http/1.1)}{localhost:37145} 2024-11-28T02:15:23,903 INFO [Time-limited test {}] server.Server(415): Started @15253ms 2024-11-28T02:15:24,035 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-28T02:15:24,039 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:24,057 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-28T02:15:24,058 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T02:15:24,059 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T02:15:24,059 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T02:15:24,059 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T02:15:24,061 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T02:15:24,063 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@53df72c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,AVAILABLE} 2024-11-28T02:15:24,063 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74e2cc15{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-28T02:15:24,105 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T02:15:24,117 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-28T02:15:24,117 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-28T02:15:24,118 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-28T02:15:24,118 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-28T02:15:24,126 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:24,130 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:24,222 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-28T02:15:24,227 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@559a435d{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/java.io.tmpdir/jetty-localhost-43641-hadoop-yarn-common-3_4_1_jar-_-any-10677588922396258053/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-28T02:15:24,229 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5f0f2e7d{HTTP/1.1, (http/1.1)}{localhost:43641} 2024-11-28T02:15:24,229 INFO [Time-limited test {}] server.Server(415): Started @15579ms 2024-11-28T02:15:24,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-28T02:15:24,269 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:38631/backupUT 2024-11-28T02:15:24,274 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:15:24,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d9827d9b919f,43413,1732760114080 2024-11-28T02:15:24,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@484d50f4 2024-11-28T02:15:24,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:15:24,290 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:15:24,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-11-28T02:15:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-28T02:15:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T02:15:24,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 19 msec 2024-11-28T02:15:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T02:15:24,415 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-28T02:15:24,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-11-28T02:15:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-28T02:15:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-28T02:15:24,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 13 msec 2024-11-28T02:15:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-28T02:15:24,535 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-28T02:15:24,536 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-11-28T02:15:24,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-28T02:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-28T02:15:24,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 15 msec 2024-11-28T02:15:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-28T02:15:24,655 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-28T02:15:24,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-11-28T02:15:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-28T02:15:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:15:24,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 14 msec 2024-11-28T02:15:24,715 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-28T02:15:24,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T02:15:24,768 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T02:15:24,770 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:15:24,770 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-28T02:15:24,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T02:15:24,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T02:15:24,771 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T02:15:24,771 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T02:15:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T02:15:24,774 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-28T02:15:24,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1732760124269', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1732760124269 2024-11-28T02:15:24,795 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1732760124269 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:15:24,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1732760124269" procId is: 8 2024-11-28T02:15:24,798 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-28T02:15:24,801 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1732760124269 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:15:24,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741838_1014 (size=367) 2024-11-28T02:15:24,816 INFO [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df85333d0693eea3c91e4c0b6b737054, NAME => 'test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1732760124269', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:24,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741839_1015 (size=53) 2024-11-28T02:15:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-28T02:15:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-28T02:15:25,226 DEBUG [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(898): Instantiated test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:25,227 DEBUG [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(1722): Closing df85333d0693eea3c91e4c0b6b737054, disabling compactions & flushes 2024-11-28T02:15:25,227 INFO [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(1755): Closing region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:25,227 DEBUG [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:25,227 DEBUG [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. after waiting 0 ms 2024-11-28T02:15:25,227 DEBUG [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:25,227 INFO [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(1973): Closed test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:25,227 DEBUG [RegionOpenAndInit-test-1732760124269-pool-0 {}] regionserver.HRegion(1676): Region close journal for df85333d0693eea3c91e4c0b6b737054: Waiting for close lock at 1732760125227Disabling compacts and flushes for region at 1732760125227Disabling writes for close at 1732760125227Writing region close event to WAL at 1732760125227Closed at 1732760125227 2024-11-28T02:15:25,229 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1732760124269 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:15:25,235 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732760125230"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732760125230"}]},"ts":"1732760125230"} 2024-11-28T02:15:25,240 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:15:25,241 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1732760124269 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:15:25,245 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732760124269","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760125241"}]},"ts":"1732760125241"} 2024-11-28T02:15:25,249 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732760124269, state=ENABLING in hbase:meta 2024-11-28T02:15:25,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, ASSIGN}] 2024-11-28T02:15:25,252 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, ASSIGN 2024-11-28T02:15:25,253 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, ASSIGN; state=OFFLINE, location=d9827d9b919f,42127,1732760115598; forceNewPlan=false, retain=false 2024-11-28T02:15:25,405 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:25,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, ASSIGN because future has completed 2024-11-28T02:15:25,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-28T02:15:25,586 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:25,587 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T02:15:25,587 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor d9827d9b919f,42127,1732760115598.regiongroup-1 2024-11-28T02:15:25,590 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1, suffix=, logDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598, archiveDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs, maxLogs=32 2024-11-28T02:15:25,606 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760125591, exclude list is [], retry=0 2024-11-28T02:15:25,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:25,613 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760125591 2024-11-28T02:15:25,613 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:25,613 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => df85333d0693eea3c91e4c0b6b737054, NAME => 'test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:25,614 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:15:25,614 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732760124269 df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,614 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:25,614 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,614 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,616 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,619 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df85333d0693eea3c91e4c0b6b737054 columnFamilyName f 2024-11-28T02:15:25,619 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:25,620 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(327): Store=df85333d0693eea3c91e4c0b6b737054/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:25,620 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,621 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,622 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,623 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,623 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,626 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,630 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:15:25,631 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened df85333d0693eea3c91e4c0b6b737054; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11329292960, jitterRate=0.055122628808021545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T02:15:25,632 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:25,633 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for df85333d0693eea3c91e4c0b6b737054: Running coprocessor pre-open hook at 1732760125614Writing region info on filesystem at 1732760125614Initializing all the Stores at 1732760125616 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760125616Cleaning up temporary data from old regions at 1732760125623 (+7 ms)Running coprocessor post-open hooks at 1732760125632 (+9 ms)Region opened successfully at 1732760125633 (+1 ms) 2024-11-28T02:15:25,635 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., pid=10, masterSystemTime=1732760125564 2024-11-28T02:15:25,639 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:25,639 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:25,641 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=OPEN, openSeqNum=2, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:25,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:15:25,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-28T02:15:25,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 in 236 msec 2024-11-28T02:15:25,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-28T02:15:25,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, ASSIGN in 400 msec 2024-11-28T02:15:25,655 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1732760124269 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:15:25,655 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732760124269","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760125655"}]},"ts":"1732760125655"} 2024-11-28T02:15:25,658 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732760124269, state=ENABLED in hbase:meta 2024-11-28T02:15:25,660 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1732760124269 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:15:25,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1732760124269 in 872 msec 2024-11-28T02:15:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-28T02:15:25,935 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1732760124269 completed 2024-11-28T02:15:25,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f031cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:25,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:25,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:25,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:25,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@938a716, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:25,942 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:25,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a9aa885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:25,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:25,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:25,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732760124269', row='row0', locateType=CURRENT is [region=test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., hostname=d9827d9b919f,42127,1732760115598, seqNum=2] 2024-11-28T02:15:25,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42127 {}] regionserver.HRegion(8528): writing data to region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. with WAL disabled. Data may be lost in the event of a crash. 2024-11-28T02:15:26,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17327601242691', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17327601242691 2024-11-28T02:15:26,154 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17327601242691 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:15:26,154 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:26,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17327601242691" procId is: 11 2024-11-28T02:15:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:15:26,156 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17327601242691 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:15:26,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741841_1017 (size=364) 2024-11-28T02:15:26,166 INFO [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30e66b01c115b81cd680f3864aaf6f9d, NAME => 'ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17327601242691', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:26,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741842_1018 (size=50) 2024-11-28T02:15:26,174 DEBUG [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:26,174 DEBUG [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(1722): Closing 30e66b01c115b81cd680f3864aaf6f9d, disabling compactions & flushes 2024-11-28T02:15:26,174 INFO [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:26,174 DEBUG [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:26,174 DEBUG [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. after waiting 0 ms 2024-11-28T02:15:26,174 DEBUG [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:26,174 INFO [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:26,175 DEBUG [RegionOpenAndInit-ns2:test-17327601242691-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30e66b01c115b81cd680f3864aaf6f9d: Waiting for close lock at 1732760126174Disabling compacts and flushes for region at 1732760126174Disabling writes for close at 1732760126174Writing region close event to WAL at 1732760126174Closed at 1732760126174 2024-11-28T02:15:26,176 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17327601242691 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:15:26,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732760126176"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732760126176"}]},"ts":"1732760126176"} 2024-11-28T02:15:26,179 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:15:26,181 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17327601242691 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:15:26,181 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17327601242691","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760126181"}]},"ts":"1732760126181"} 2024-11-28T02:15:26,183 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17327601242691, state=ENABLING in hbase:meta 2024-11-28T02:15:26,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17327601242691, region=30e66b01c115b81cd680f3864aaf6f9d, ASSIGN}] 2024-11-28T02:15:26,185 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17327601242691, region=30e66b01c115b81cd680f3864aaf6f9d, ASSIGN 2024-11-28T02:15:26,186 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17327601242691, region=30e66b01c115b81cd680f3864aaf6f9d, ASSIGN; state=OFFLINE, location=d9827d9b919f,42127,1732760115598; forceNewPlan=false, retain=false 2024-11-28T02:15:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:15:26,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=30e66b01c115b81cd680f3864aaf6f9d, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:26,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17327601242691, region=30e66b01c115b81cd680f3864aaf6f9d, ASSIGN because future has completed 2024-11-28T02:15:26,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:15:26,500 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:26,500 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 30e66b01c115b81cd680f3864aaf6f9d, NAME => 'ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:26,501 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:15:26,501 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17327601242691 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,501 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:26,501 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,501 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,503 INFO [StoreOpener-30e66b01c115b81cd680f3864aaf6f9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,506 INFO [StoreOpener-30e66b01c115b81cd680f3864aaf6f9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30e66b01c115b81cd680f3864aaf6f9d columnFamilyName f 2024-11-28T02:15:26,506 DEBUG [StoreOpener-30e66b01c115b81cd680f3864aaf6f9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:26,507 INFO [StoreOpener-30e66b01c115b81cd680f3864aaf6f9d-1 {}] regionserver.HStore(327): Store=30e66b01c115b81cd680f3864aaf6f9d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:26,507 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,509 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,509 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,510 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,510 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,514 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,517 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:15:26,518 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 30e66b01c115b81cd680f3864aaf6f9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11040929440, jitterRate=0.028266683220863342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T02:15:26,518 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:26,519 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 30e66b01c115b81cd680f3864aaf6f9d: Running coprocessor pre-open hook at 1732760126501Writing region info on filesystem at 1732760126501Initializing all the Stores at 1732760126503 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760126503Cleaning up temporary data from old regions at 1732760126510 (+7 ms)Running coprocessor post-open hooks at 1732760126518 (+8 ms)Region opened successfully at 1732760126519 (+1 ms) 2024-11-28T02:15:26,520 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d., pid=13, masterSystemTime=1732760126495 2024-11-28T02:15:26,523 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:26,523 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:26,524 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=30e66b01c115b81cd680f3864aaf6f9d, regionState=OPEN, openSeqNum=2, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:26,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:15:26,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-28T02:15:26,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d, server=d9827d9b919f,42127,1732760115598 in 188 msec 2024-11-28T02:15:26,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T02:15:26,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17327601242691, region=30e66b01c115b81cd680f3864aaf6f9d, ASSIGN in 349 msec 2024-11-28T02:15:26,538 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17327601242691 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:15:26,538 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17327601242691","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760126538"}]},"ts":"1732760126538"} 2024-11-28T02:15:26,541 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17327601242691, state=ENABLED in hbase:meta 2024-11-28T02:15:26,542 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17327601242691 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:15:26,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17327601242691 in 391 msec 2024-11-28T02:15:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T02:15:26,784 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17327601242691 completed 2024-11-28T02:15:26,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17327601242691', row='row0', locateType=CURRENT is [region=ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d., hostname=d9827d9b919f,42127,1732760115598, seqNum=2] 2024-11-28T02:15:26,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42127 {}] regionserver.HRegion(8528): writing data to region ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. with WAL disabled. Data may be lost in the event of a crash. 2024-11-28T02:15:26,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17327601242692', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17327601242692 2024-11-28T02:15:26,926 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17327601242692 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:15:26,927 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:26,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17327601242692" procId is: 14 2024-11-28T02:15:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-28T02:15:26,928 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17327601242692 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:15:26,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741843_1019 (size=390) 2024-11-28T02:15:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-28T02:15:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-28T02:15:27,339 INFO [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 795ba51d3cf9aa45e08a6768f6136859, NAME => 'ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17327601242692', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:27,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741844_1020 (size=50) 2024-11-28T02:15:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-28T02:15:27,748 DEBUG [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:27,748 DEBUG [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(1722): Closing 795ba51d3cf9aa45e08a6768f6136859, disabling compactions & flushes 2024-11-28T02:15:27,748 INFO [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:15:27,748 DEBUG [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:15:27,748 DEBUG [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. after waiting 0 ms 2024-11-28T02:15:27,748 DEBUG [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:15:27,748 INFO [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:15:27,748 DEBUG [RegionOpenAndInit-ns3:test-17327601242692-pool-0 {}] regionserver.HRegion(1676): Region close journal for 795ba51d3cf9aa45e08a6768f6136859: Waiting for close lock at 1732760127748Disabling compacts and flushes for region at 1732760127748Disabling writes for close at 1732760127748Writing region close event to WAL at 1732760127748Closed at 1732760127748 2024-11-28T02:15:27,750 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17327601242692 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:15:27,751 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732760127750"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732760127750"}]},"ts":"1732760127750"} 2024-11-28T02:15:27,753 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:15:27,754 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17327601242692 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:15:27,755 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17327601242692","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760127754"}]},"ts":"1732760127754"} 2024-11-28T02:15:27,757 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17327601242692, state=ENABLING in hbase:meta 2024-11-28T02:15:27,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17327601242692, region=795ba51d3cf9aa45e08a6768f6136859, ASSIGN}] 2024-11-28T02:15:27,758 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17327601242692, region=795ba51d3cf9aa45e08a6768f6136859, ASSIGN 2024-11-28T02:15:27,759 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17327601242692, region=795ba51d3cf9aa45e08a6768f6136859, ASSIGN; state=OFFLINE, location=d9827d9b919f,42127,1732760115598; forceNewPlan=false, retain=false 2024-11-28T02:15:27,910 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=795ba51d3cf9aa45e08a6768f6136859, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:27,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17327601242692, region=795ba51d3cf9aa45e08a6768f6136859, ASSIGN because future has completed 2024-11-28T02:15:27,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 795ba51d3cf9aa45e08a6768f6136859, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-28T02:15:28,073 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:15:28,073 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => 795ba51d3cf9aa45e08a6768f6136859, NAME => 'ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:28,073 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:15:28,073 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17327601242692 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,073 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:28,074 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,074 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,075 INFO [StoreOpener-795ba51d3cf9aa45e08a6768f6136859-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,078 INFO [StoreOpener-795ba51d3cf9aa45e08a6768f6136859-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 795ba51d3cf9aa45e08a6768f6136859 columnFamilyName f 2024-11-28T02:15:28,078 DEBUG [StoreOpener-795ba51d3cf9aa45e08a6768f6136859-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:28,079 INFO [StoreOpener-795ba51d3cf9aa45e08a6768f6136859-1 {}] regionserver.HStore(327): Store=795ba51d3cf9aa45e08a6768f6136859/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:28,079 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,080 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns3/test-17327601242692/795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,081 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns3/test-17327601242692/795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,081 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,081 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,084 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,087 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns3/test-17327601242692/795ba51d3cf9aa45e08a6768f6136859/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:15:28,088 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened 795ba51d3cf9aa45e08a6768f6136859; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11776334400, jitterRate=0.09675660729408264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T02:15:28,088 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:15:28,089 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for 795ba51d3cf9aa45e08a6768f6136859: Running coprocessor pre-open hook at 1732760128074Writing region info on filesystem at 1732760128074Initializing all the Stores at 1732760128075 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760128075Cleaning up temporary data from old regions at 1732760128081 (+6 ms)Running coprocessor post-open hooks at 1732760128088 (+7 ms)Region opened successfully at 1732760128089 (+1 ms) 2024-11-28T02:15:28,090 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859., pid=16, masterSystemTime=1732760128067 2024-11-28T02:15:28,093 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:15:28,093 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:15:28,094 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=795ba51d3cf9aa45e08a6768f6136859, regionState=OPEN, openSeqNum=2, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:28,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 795ba51d3cf9aa45e08a6768f6136859, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:15:28,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-28T02:15:28,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 795ba51d3cf9aa45e08a6768f6136859, server=d9827d9b919f,42127,1732760115598 in 183 msec 2024-11-28T02:15:28,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-28T02:15:28,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17327601242692, region=795ba51d3cf9aa45e08a6768f6136859, ASSIGN in 343 msec 2024-11-28T02:15:28,104 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17327601242692 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:15:28,104 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17327601242692","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760128104"}]},"ts":"1732760128104"} 2024-11-28T02:15:28,106 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17327601242692, state=ENABLED in hbase:meta 2024-11-28T02:15:28,107 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17327601242692 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:15:28,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17327601242692 in 1.1840 sec 2024-11-28T02:15:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-28T02:15:29,084 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17327601242692 completed 2024-11-28T02:15:29,084 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17327601242692 get assigned. Timeout = 60000ms 2024-11-28T02:15:29,086 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T02:15:29,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17327601242692 assigned to meta. Checking AM states. 2024-11-28T02:15:29,092 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T02:15:29,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17327601242692 assigned. 2024-11-28T02:15:29,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17327601242693', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17327601242693 2024-11-28T02:15:29,098 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17327601242693 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:15:29,098 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:29,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17327601242693" procId is: 17 2024-11-28T02:15:29,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:15:29,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17327601242693 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:15:29,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741845_1021 (size=390) 2024-11-28T02:15:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:15:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:15:29,511 INFO [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d29139dea3e9e00e0b5beeb2ae62089f, NAME => 'ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17327601242693', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:29,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741846_1022 (size=50) 2024-11-28T02:15:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:15:29,921 DEBUG [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:29,921 DEBUG [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(1722): Closing d29139dea3e9e00e0b5beeb2ae62089f, disabling compactions & flushes 2024-11-28T02:15:29,921 INFO [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:15:29,921 DEBUG [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:15:29,921 DEBUG [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. after waiting 0 ms 2024-11-28T02:15:29,921 DEBUG [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:15:29,921 INFO [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:15:29,921 DEBUG [RegionOpenAndInit-ns4:test-17327601242693-pool-0 {}] regionserver.HRegion(1676): Region close journal for d29139dea3e9e00e0b5beeb2ae62089f: Waiting for close lock at 1732760129921Disabling compacts and flushes for region at 1732760129921Disabling writes for close at 1732760129921Writing region close event to WAL at 1732760129921Closed at 1732760129921 2024-11-28T02:15:29,924 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17327601242693 execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:15:29,924 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732760129924"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732760129924"}]},"ts":"1732760129924"} 2024-11-28T02:15:29,927 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:15:29,928 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17327601242693 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:15:29,928 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17327601242693","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760129928"}]},"ts":"1732760129928"} 2024-11-28T02:15:29,931 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17327601242693, state=ENABLING in hbase:meta 2024-11-28T02:15:29,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17327601242693, region=d29139dea3e9e00e0b5beeb2ae62089f, ASSIGN}] 2024-11-28T02:15:29,932 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17327601242693, region=d29139dea3e9e00e0b5beeb2ae62089f, ASSIGN 2024-11-28T02:15:29,934 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17327601242693, region=d29139dea3e9e00e0b5beeb2ae62089f, ASSIGN; state=OFFLINE, location=d9827d9b919f,42127,1732760115598; forceNewPlan=false, retain=false 2024-11-28T02:15:30,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=d29139dea3e9e00e0b5beeb2ae62089f, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:30,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17327601242693, region=d29139dea3e9e00e0b5beeb2ae62089f, ASSIGN because future has completed 2024-11-28T02:15:30,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure d29139dea3e9e00e0b5beeb2ae62089f, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:30,110 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17327601242692' 2024-11-28T02:15:30,111 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17327601242691' 2024-11-28T02:15:30,111 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1732760124269' 2024-11-28T02:15:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:15:30,245 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:15:30,246 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => d29139dea3e9e00e0b5beeb2ae62089f, NAME => 'ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:30,246 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:15:30,246 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17327601242693 d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,246 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:30,246 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,246 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,248 INFO [StoreOpener-d29139dea3e9e00e0b5beeb2ae62089f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,251 INFO [StoreOpener-d29139dea3e9e00e0b5beeb2ae62089f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d29139dea3e9e00e0b5beeb2ae62089f columnFamilyName f 2024-11-28T02:15:30,251 DEBUG [StoreOpener-d29139dea3e9e00e0b5beeb2ae62089f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:30,252 INFO [StoreOpener-d29139dea3e9e00e0b5beeb2ae62089f-1 {}] regionserver.HStore(327): Store=d29139dea3e9e00e0b5beeb2ae62089f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:30,252 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,253 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns4/test-17327601242693/d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,253 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns4/test-17327601242693/d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,254 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,254 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,256 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,258 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns4/test-17327601242693/d29139dea3e9e00e0b5beeb2ae62089f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:15:30,259 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened d29139dea3e9e00e0b5beeb2ae62089f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10493967680, jitterRate=-0.022673100233078003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T02:15:30,259 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:15:30,260 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for d29139dea3e9e00e0b5beeb2ae62089f: Running coprocessor pre-open hook at 1732760130246Writing region info on filesystem at 1732760130247 (+1 ms)Initializing all the Stores at 1732760130248 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760130248Cleaning up temporary data from old regions at 1732760130254 (+6 ms)Running coprocessor post-open hooks at 1732760130259 (+5 ms)Region opened successfully at 1732760130260 (+1 ms) 2024-11-28T02:15:30,261 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f., pid=19, masterSystemTime=1732760130241 2024-11-28T02:15:30,263 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:15:30,264 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:15:30,264 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=d29139dea3e9e00e0b5beeb2ae62089f, regionState=OPEN, openSeqNum=2, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:30,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure d29139dea3e9e00e0b5beeb2ae62089f, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:15:30,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-28T02:15:30,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure d29139dea3e9e00e0b5beeb2ae62089f, server=d9827d9b919f,42127,1732760115598 in 180 msec 2024-11-28T02:15:30,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T02:15:30,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17327601242693, region=d29139dea3e9e00e0b5beeb2ae62089f, ASSIGN in 340 msec 2024-11-28T02:15:30,275 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17327601242693 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:15:30,275 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17327601242693","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760130275"}]},"ts":"1732760130275"} 2024-11-28T02:15:30,277 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17327601242693, state=ENABLED in hbase:meta 2024-11-28T02:15:30,278 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17327601242693 execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:15:30,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17327601242693 in 1.1830 sec 2024-11-28T02:15:31,117 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-28T02:15:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T02:15:31,244 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17327601242693 completed 2024-11-28T02:15:31,244 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17327601242693 get assigned. Timeout = 60000ms 2024-11-28T02:15:31,245 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T02:15:31,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17327601242693 assigned to meta. Checking AM states. 2024-11-28T02:15:31,250 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T02:15:31,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17327601242693 assigned. 2024-11-28T02:15:31,250 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:15:31,250 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:15:31,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:31,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:31,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:15:31,288 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=107, ProcessCount=14, AvailableMemoryMB=7856 2024-11-28T02:15:31,290 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-11-28T02:15:31,292 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-28T02:15:31,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732760124269 from 'test-1732760124269', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732760124269', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732760124269 2024-11-28T02:15:31,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741847_1023 (size=908) 2024-11-28T02:15:31,331 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/.tabledesc/.tableinfo.0000000001.367 2024-11-28T02:15:31,331 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/.tabledesc/.tableinfo.0000000002.908 2024-11-28T02:15:31,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732760124269}] 2024-11-28T02:15:31,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE}] 2024-11-28T02:15:31,349 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE 2024-11-28T02:15:31,351 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=CLOSING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:31,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE because future has completed 2024-11-28T02:15:31,354 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:15:31,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:31,514 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:31,514 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:15:31,515 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing df85333d0693eea3c91e4c0b6b737054, disabling compactions & flushes 2024-11-28T02:15:31,515 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:31,515 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:31,515 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. after waiting 0 ms 2024-11-28T02:15:31,515 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:31,519 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing df85333d0693eea3c91e4c0b6b737054 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-28T02:15:31,587 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f/001e01fc909c46b5bb46d75f30451132 is 37, key is row10/f:q1/1732760125978/Put/seqid=0 2024-11-28T02:15:31,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741848_1024 (size=8413) 2024-11-28T02:15:32,000 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f/001e01fc909c46b5bb46d75f30451132 2024-11-28T02:15:32,063 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f/001e01fc909c46b5bb46d75f30451132 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 2024-11-28T02:15:32,072 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132, entries=99, sequenceid=103, filesize=8.2 K 2024-11-28T02:15:32,079 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for df85333d0693eea3c91e4c0b6b737054 in 560ms, sequenceid=103, compaction requested=false 2024-11-28T02:15:32,085 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-28T02:15:32,088 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:15:32,089 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:32,089 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for df85333d0693eea3c91e4c0b6b737054: Waiting for close lock at 1732760131515Running coprocessor pre-close hooks at 1732760131515Disabling compacts and flushes for region at 1732760131515Disabling writes for close at 1732760131515Obtaining lock to block concurrent updates at 1732760131519 (+4 ms)Preparing flush snapshotting stores in df85333d0693eea3c91e4c0b6b737054 at 1732760131519Finished memstore snapshotting test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1732760131527 (+8 ms)Flushing stores of test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. at 1732760131528 (+1 ms)Flushing df85333d0693eea3c91e4c0b6b737054/f: creating writer at 1732760131530 (+2 ms)Flushing df85333d0693eea3c91e4c0b6b737054/f: appending metadata at 1732760131576 (+46 ms)Flushing df85333d0693eea3c91e4c0b6b737054/f: closing flushed file at 1732760131579 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@309b0ea9: reopening flushed file at 1732760132061 (+482 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for df85333d0693eea3c91e4c0b6b737054 in 560ms, sequenceid=103, compaction requested=false at 1732760132079 (+18 ms)Writing region close event to WAL at 1732760132080 (+1 ms)Running coprocessor post-close hooks at 1732760132086 (+6 ms)Closed at 1732760132088 (+2 ms) 2024-11-28T02:15:32,089 WARN [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: df85333d0693eea3c91e4c0b6b737054 to self. 2024-11-28T02:15:32,092 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,093 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=CLOSED 2024-11-28T02:15:32,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:15:32,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-28T02:15:32,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 in 742 msec 2024-11-28T02:15:32,100 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE; state=CLOSED, location=d9827d9b919f,42127,1732760115598; forceNewPlan=false, retain=true 2024-11-28T02:15:32,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:32,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE because future has completed 2024-11-28T02:15:32,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:32,414 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:32,414 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => df85333d0693eea3c91e4c0b6b737054, NAME => 'test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:32,415 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:15:32,415 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732760124269 df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,415 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:32,415 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,415 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,417 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,418 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df85333d0693eea3c91e4c0b6b737054 columnFamilyName f 2024-11-28T02:15:32,418 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:32,428 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 2024-11-28T02:15:32,428 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(327): Store=df85333d0693eea3c91e4c0b6b737054/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:32,428 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,430 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df85333d0693eea3c91e4c0b6b737054 columnFamilyName f3 2024-11-28T02:15:32,430 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:32,431 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(327): Store=df85333d0693eea3c91e4c0b6b737054/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:32,432 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,438 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df85333d0693eea3c91e4c0b6b737054 columnFamilyName mob 2024-11-28T02:15:32,443 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:32,443 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(327): Store=df85333d0693eea3c91e4c0b6b737054/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:32,444 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,445 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,447 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,449 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,449 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,450 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732760124269 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-28T02:15:32,452 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,454 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened df85333d0693eea3c91e4c0b6b737054; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10880156640, jitterRate=0.013293549418449402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-28T02:15:32,454 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:32,455 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for df85333d0693eea3c91e4c0b6b737054: Running coprocessor pre-open hook at 1732760132415Writing region info on filesystem at 1732760132415Initializing all the Stores at 1732760132416 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760132417 (+1 ms)Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760132417Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760132417Cleaning up temporary data from old regions at 1732760132449 (+32 ms)Running coprocessor post-open hooks at 1732760132454 (+5 ms)Region opened successfully at 1732760132455 (+1 ms) 2024-11-28T02:15:32,456 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., pid=24, masterSystemTime=1732760132408 2024-11-28T02:15:32,459 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:32,459 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:32,460 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=OPEN, openSeqNum=107, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:32,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:15:32,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-28T02:15:32,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 in 209 msec 2024-11-28T02:15:32,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-28T02:15:32,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE in 1.1200 sec 2024-11-28T02:15:32,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-28T02:15:32,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732760124269 in 1.1350 sec 2024-11-28T02:15:32,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732760124269 in 1.1760 sec 2024-11-28T02:15:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-28T02:15:32,482 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732760124269 completed 2024-11-28T02:15:32,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f03fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:32,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:32,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:32,485 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:32,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:32,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:32,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9adeda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:32,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:32,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:32,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:32,487 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:32,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cadcbe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:32,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:32,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:32,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:32,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:32,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732760124269', row='row-test-1732760124269-3-0', locateType=CURRENT is [region=test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., hostname=d9827d9b919f,42127,1732760115598, seqNum=107] 2024-11-28T02:15:32,539 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:15:32,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is d9827d9b919f,43413,1732760114080 2024-11-28T02:15:32,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@367ad936 2024-11-28T02:15:32,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:15:32,545 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:15:32,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-11-28T02:15:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-28T02:15:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:15:32,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 9 msec 2024-11-28T02:15:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T02:15:32,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-28T02:15:32,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-28T02:15:32,685 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:15:32,685 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:32,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-28T02:15:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-28T02:15:32,686 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:15:32,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741849_1025 (size=635) 2024-11-28T02:15:32,697 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a2d65c385b1b9e1c5b04dfa38326a47a, NAME => 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:32,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741850_1026 (size=40) 2024-11-28T02:15:32,704 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:32,704 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing a2d65c385b1b9e1c5b04dfa38326a47a, disabling compactions & flushes 2024-11-28T02:15:32,704 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:32,704 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:32,704 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. after waiting 0 ms 2024-11-28T02:15:32,704 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:32,704 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:32,705 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for a2d65c385b1b9e1c5b04dfa38326a47a: Waiting for close lock at 1732760132704Disabling compacts and flushes for region at 1732760132704Disabling writes for close at 1732760132704Writing region close event to WAL at 1732760132704Closed at 1732760132704 2024-11-28T02:15:32,706 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:15:32,707 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732760132706"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732760132706"}]},"ts":"1732760132706"} 2024-11-28T02:15:32,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:15:32,710 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:15:32,710 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760132710"}]},"ts":"1732760132710"} 2024-11-28T02:15:32,712 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-28T02:15:32,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN}] 2024-11-28T02:15:32,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN 2024-11-28T02:15:32,715 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN; state=OFFLINE, location=d9827d9b919f,42127,1732760115598; forceNewPlan=false, retain=false 2024-11-28T02:15:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-28T02:15:32,866 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a2d65c385b1b9e1c5b04dfa38326a47a, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:32,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN because future has completed 2024-11-28T02:15:32,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-28T02:15:33,027 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:33,027 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => a2d65c385b1b9e1c5b04dfa38326a47a, NAME => 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:33,028 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:15:33,028 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,028 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:33,029 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,029 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,031 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,033 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2d65c385b1b9e1c5b04dfa38326a47a columnFamilyName meta 2024-11-28T02:15:33,033 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:33,034 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.HStore(327): Store=a2d65c385b1b9e1c5b04dfa38326a47a/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:33,034 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,036 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2d65c385b1b9e1c5b04dfa38326a47a columnFamilyName session 2024-11-28T02:15:33,036 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:33,036 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.HStore(327): Store=a2d65c385b1b9e1c5b04dfa38326a47a/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:33,037 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,038 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,038 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,040 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,040 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,041 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-28T02:15:33,043 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,047 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:15:33,048 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened a2d65c385b1b9e1c5b04dfa38326a47a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11651337280, jitterRate=0.08511534333229065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-28T02:15:33,048 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:33,049 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for a2d65c385b1b9e1c5b04dfa38326a47a: Running coprocessor pre-open hook at 1732760133029Writing region info on filesystem at 1732760133029Initializing all the Stores at 1732760133030 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760133030Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760133030Cleaning up temporary data from old regions at 1732760133040 (+10 ms)Running coprocessor post-open hooks at 1732760133048 (+8 ms)Region opened successfully at 1732760133049 (+1 ms) 2024-11-28T02:15:33,050 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., pid=28, masterSystemTime=1732760133022 2024-11-28T02:15:33,053 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:33,053 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:33,053 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a2d65c385b1b9e1c5b04dfa38326a47a, regionState=OPEN, openSeqNum=2, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:33,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:15:33,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-28T02:15:33,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598 in 188 msec 2024-11-28T02:15:33,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-28T02:15:33,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN in 347 msec 2024-11-28T02:15:33,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:15:33,064 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760133064"}]},"ts":"1732760133064"} 2024-11-28T02:15:33,066 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-28T02:15:33,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:15:33,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 385 msec 2024-11-28T02:15:33,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-28T02:15:33,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-28T02:15:33,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:33,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-28T02:15:33,335 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T02:15:33,335 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:33,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-28T02:15:33,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T02:15:33,337 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T02:15:33,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741851_1027 (size=640) 2024-11-28T02:15:33,346 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 63374509f749d336450706661d23059d, NAME => 'backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:33,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741852_1028 (size=45) 2024-11-28T02:15:33,355 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:33,355 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing 63374509f749d336450706661d23059d, disabling compactions & flushes 2024-11-28T02:15:33,355 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:15:33,355 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:15:33,355 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. after waiting 0 ms 2024-11-28T02:15:33,355 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:15:33,355 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:15:33,356 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for 63374509f749d336450706661d23059d: Waiting for close lock at 1732760133355Disabling compacts and flushes for region at 1732760133355Disabling writes for close at 1732760133355Writing region close event to WAL at 1732760133355Closed at 1732760133355 2024-11-28T02:15:33,357 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T02:15:33,357 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732760133357"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732760133357"}]},"ts":"1732760133357"} 2024-11-28T02:15:33,360 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:15:33,361 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T02:15:33,361 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760133361"}]},"ts":"1732760133361"} 2024-11-28T02:15:33,363 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-28T02:15:33,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=63374509f749d336450706661d23059d, ASSIGN}] 2024-11-28T02:15:33,365 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=63374509f749d336450706661d23059d, ASSIGN 2024-11-28T02:15:33,366 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=63374509f749d336450706661d23059d, ASSIGN; state=OFFLINE, location=d9827d9b919f,42127,1732760115598; forceNewPlan=false, retain=false 2024-11-28T02:15:33,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T02:15:33,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=63374509f749d336450706661d23059d, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:33,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=63374509f749d336450706661d23059d, ASSIGN because future has completed 2024-11-28T02:15:33,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63374509f749d336450706661d23059d, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:15:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T02:15:33,678 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:15:33,678 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 63374509f749d336450706661d23059d, NAME => 'backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:15:33,679 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:15:33,679 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 63374509f749d336450706661d23059d 2024-11-28T02:15:33,679 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:15:33,679 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 63374509f749d336450706661d23059d 2024-11-28T02:15:33,679 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 63374509f749d336450706661d23059d 2024-11-28T02:15:33,681 INFO [StoreOpener-63374509f749d336450706661d23059d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 63374509f749d336450706661d23059d 2024-11-28T02:15:33,683 INFO [StoreOpener-63374509f749d336450706661d23059d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63374509f749d336450706661d23059d columnFamilyName meta 2024-11-28T02:15:33,683 DEBUG [StoreOpener-63374509f749d336450706661d23059d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:33,684 INFO [StoreOpener-63374509f749d336450706661d23059d-1 {}] regionserver.HStore(327): Store=63374509f749d336450706661d23059d/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:33,684 INFO [StoreOpener-63374509f749d336450706661d23059d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 63374509f749d336450706661d23059d 2024-11-28T02:15:33,685 INFO [StoreOpener-63374509f749d336450706661d23059d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63374509f749d336450706661d23059d columnFamilyName session 2024-11-28T02:15:33,685 DEBUG [StoreOpener-63374509f749d336450706661d23059d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:33,686 INFO [StoreOpener-63374509f749d336450706661d23059d-1 {}] regionserver.HStore(327): Store=63374509f749d336450706661d23059d/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:15:33,686 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 63374509f749d336450706661d23059d 2024-11-28T02:15:33,687 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system_bulk/63374509f749d336450706661d23059d 2024-11-28T02:15:33,687 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system_bulk/63374509f749d336450706661d23059d 2024-11-28T02:15:33,688 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 63374509f749d336450706661d23059d 2024-11-28T02:15:33,688 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 63374509f749d336450706661d23059d 2024-11-28T02:15:33,689 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-28T02:15:33,691 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 63374509f749d336450706661d23059d 2024-11-28T02:15:33,694 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system_bulk/63374509f749d336450706661d23059d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T02:15:33,695 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 63374509f749d336450706661d23059d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10569616960, jitterRate=-0.01562771201133728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-28T02:15:33,695 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63374509f749d336450706661d23059d 2024-11-28T02:15:33,696 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 63374509f749d336450706661d23059d: Running coprocessor pre-open hook at 1732760133679Writing region info on filesystem at 1732760133679Initializing all the Stores at 1732760133681 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760133681Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760133681Cleaning up temporary data from old regions at 1732760133688 (+7 ms)Running coprocessor post-open hooks at 1732760133695 (+7 ms)Region opened successfully at 1732760133696 (+1 ms) 2024-11-28T02:15:33,697 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d., pid=31, masterSystemTime=1732760133673 2024-11-28T02:15:33,700 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:15:33,700 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:15:33,700 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=63374509f749d336450706661d23059d, regionState=OPEN, openSeqNum=2, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:33,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63374509f749d336450706661d23059d, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:15:33,704 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43413 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d9827d9b919f,42127,1732760115598, table=backup:system_bulk, region=63374509f749d336450706661d23059d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-28T02:15:33,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-28T02:15:33,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 63374509f749d336450706661d23059d, server=d9827d9b919f,42127,1732760115598 in 184 msec 2024-11-28T02:15:33,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-28T02:15:33,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=63374509f749d336450706661d23059d, ASSIGN in 343 msec 2024-11-28T02:15:33,711 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T02:15:33,712 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760133711"}]},"ts":"1732760133711"} 2024-11-28T02:15:33,714 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-28T02:15:33,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T02:15:33,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 383 msec 2024-11-28T02:15:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-28T02:15:33,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-28T02:15:33,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:15:33,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:15:33,985 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-28T02:15:33,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., hostname=d9827d9b919f,42127,1732760115598, seqNum=2] 2024-11-28T02:15:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42127 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-28T02:15:34,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:15:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760134011 (current time:1732760134011). 2024-11-28T02:15:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:15:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-28T02:15:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77abf7f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:34,013 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:34,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:34,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:34,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c6d9bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:34,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:34,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:34,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:34,016 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:34,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb9a8de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:34,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:34,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:34,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:34,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:15:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:15:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:34,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:15:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:15:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:15:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-28T02:15:34,035 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:15:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T02:15:34,040 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:15:34,052 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:15:34,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741853_1029 (size=52) 2024-11-28T02:15:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T02:15:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T02:15:34,462 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:15:34,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a}] 2024-11-28T02:15:34,470 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:34,508 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-28T02:15:34,509 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-28T02:15:34,510 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17327601242691 2024-11-28T02:15:34,510 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17327601242691 Metrics about Tables on a single HBase RegionServer 2024-11-28T02:15:34,511 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17327601242693 2024-11-28T02:15:34,511 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17327601242693 Metrics about Tables on a single HBase RegionServer 2024-11-28T02:15:34,512 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-28T02:15:34,512 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-28T02:15:34,513 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732760124269 2024-11-28T02:15:34,513 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1732760124269 Metrics about Tables on a single HBase RegionServer 2024-11-28T02:15:34,514 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17327601242692 2024-11-28T02:15:34,514 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17327601242692 Metrics about Tables on a single HBase RegionServer 2024-11-28T02:15:34,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-28T02:15:34,628 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:34,629 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing a2d65c385b1b9e1c5b04dfa38326a47a 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-28T02:15:34,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/44177ac9f4e946898d32e4854b5d17ba is 49, key is activesession:/session:c/1732760134001/Put/seqid=0 2024-11-28T02:15:34,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741854_1030 (size=5049) 2024-11-28T02:15:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T02:15:35,055 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/44177ac9f4e946898d32e4854b5d17ba 2024-11-28T02:15:35,065 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/44177ac9f4e946898d32e4854b5d17ba as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba 2024-11-28T02:15:35,073 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba, entries=1, sequenceid=5, filesize=4.9 K 2024-11-28T02:15:35,074 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for a2d65c385b1b9e1c5b04dfa38326a47a in 446ms, sequenceid=5, compaction requested=false 2024-11-28T02:15:35,075 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-28T02:15:35,075 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for a2d65c385b1b9e1c5b04dfa38326a47a: 2024-11-28T02:15:35,075 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. for snapshot_backup_system completed. 2024-11-28T02:15:35,076 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.' region-info for snapshot=snapshot_backup_system 2024-11-28T02:15:35,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:15:35,082 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-28T02:15:35,082 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba] hfiles 2024-11-28T02:15:35,084 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba for snapshot=snapshot_backup_system 2024-11-28T02:15:35,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741855_1031 (size=95) 2024-11-28T02:15:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T02:15:35,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:35,496 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-28T02:15:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-28T02:15:35,499 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:35,500 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:35,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-28T02:15:35,504 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:15:35,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a in 1.0360 sec 2024-11-28T02:15:35,506 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:15:35,507 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:15:35,507 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-28T02:15:35,509 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-28T02:15:35,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741856_1032 (size=731) 2024-11-28T02:15:35,927 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:15:35,939 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:15:35,940 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_backup_system 2024-11-28T02:15:35,942 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:15:35,942 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-28T02:15:35,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9130 sec 2024-11-28T02:15:36,113 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-28T02:15:36,113 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17327601242693' 2024-11-28T02:15:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T02:15:36,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-28T02:15:36,185 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732760132524 started at 1732760136184. 2024-11-28T02:15:36,199 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732760132524 has been started. 2024-11-28T02:15:36,206 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-28T02:15:36,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-28T02:15:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-28T02:15:36,215 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-28T02:15:36,215 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-28T02:15:36,215 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-28T02:15:36,215 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-28T02:15:36,216 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,216 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:36,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-28T02:15:36,218 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,219 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-28T02:15:36,219 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:15:36,219 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,219 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-28T02:15:36,220 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:36,220 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,221 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-28T02:15:36,221 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:36,221 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-28T02:15:36,223 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-28T02:15:36,223 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-28T02:15:36,224 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-28T02:15:36,224 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-28T02:15:36,224 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-28T02:15:36,224 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-28T02:15:36,224 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'd9827d9b919f,42127,1732760115598' joining acquired barrier for procedure (rolllog) in zk 2024-11-28T02:15:36,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,227 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:36,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:15:36,227 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-28T02:15:36,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:15:36,227 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-28T02:15:36,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:15:36,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:15:36,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:36,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:15:36,230 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'd9827d9b919f,42127,1732760115598' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-28T02:15:36,230 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-28T02:15:36,230 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@62b146d7[Count = 0] remaining members to acquire global barrier 2024-11-28T02:15:36,231 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:36,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:36,232 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:36,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:36,233 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,233 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-28T02:15:36,233 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-28T02:15:36,233 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-28T02:15:36,234 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,241 INFO [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732760118299 highest: 1732760125591 on d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,242 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.meta:.meta(num 1732760119169) roll requested 2024-11-28T02:15:36,255 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta, exclude list is [], retry=0 2024-11-28T02:15:36,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:36,263 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760119169.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta 2024-11-28T02:15:36,263 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:36,264 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760119169.meta is not closed yet, will try archiving it next time 2024-11-28T02:15:36,264 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1:(num 1732760125591) roll requested 2024-11-28T02:15:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741834_1010 (size=12055) 2024-11-28T02:15:36,267 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760119169.meta not finished, retry = 0 2024-11-28T02:15:36,279 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264, exclude list is [], retry=0 2024-11-28T02:15:36,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:36,284 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760125591 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 2024-11-28T02:15:36,284 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:36,284 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760125591 is not closed yet, will try archiving it next time 2024-11-28T02:15:36,284 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0:(num 1732760118299) roll requested 2024-11-28T02:15:36,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741840_1016 (size=4756) 2024-11-28T02:15:36,286 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760125591 not finished, retry = 0 2024-11-28T02:15:36,298 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284, exclude list is [], retry=0 2024-11-28T02:15:36,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:36,302 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-28T02:15:36,302 INFO [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732760136284 on d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,303 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:15:36,303 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is d9827d9b919f,43413,1732760114080 2024-11-28T02:15:36,303 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4bf5d8d8 2024-11-28T02:15:36,303 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:15:36,303 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760118299 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 2024-11-28T02:15:36,304 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:36,304 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760118299 is not closed yet, will try archiving it next time 2024-11-28T02:15:36,304 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33161, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-28T02:15:36,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741832_1008 (size=1034) 2024-11-28T02:15:36,306 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:36,306 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:36,306 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:36,307 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760118299 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760118299 2024-11-28T02:15:36,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53835, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-28T02:15:36,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:15:36,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:15:36,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:38631/backupUT\x00', locateType=CURRENT is [region=backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., hostname=d9827d9b919f,42127,1732760115598, seqNum=2] 2024-11-28T02:15:36,336 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-28T02:15:36,336 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'd9827d9b919f,42127,1732760115598' in zk 2024-11-28T02:15:36,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,339 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-28T02:15:36,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,339 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-28T02:15:36,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:15:36,339 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:15:36,340 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-28T02:15:36,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:15:36,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:15:36,341 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:36,341 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,342 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:15:36,342 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:36,343 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,344 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'd9827d9b919f,42127,1732760115598': 2024-11-28T02:15:36,344 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-28T02:15:36,344 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-28T02:15:36,344 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-28T02:15:36,344 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-28T02:15:36,344 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-28T02:15:36,344 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'd9827d9b919f,42127,1732760115598' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-28T02:15:36,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-28T02:15:36,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,347 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:15:36,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:15:36,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:15:36,347 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-28T02:15:36,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:15:36,348 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,348 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:36,348 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,348 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:15:36,349 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:36,350 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,350 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:15:36,350 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,350 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:36,351 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-28T02:15:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:36,358 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-28T02:15:36,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:15:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-28T02:15:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:15:36,358 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-28T02:15:36,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-28T02:15:36,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-28T02:15:36,359 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-28T02:15:36,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:15:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-28T02:15:36,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } 2024-11-28T02:15:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760136370 (current time:1732760136370). 2024-11-28T02:15:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:15:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732760136370_default_test-1732760124269 VERSION not specified, setting to 2 2024-11-28T02:15:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e9d97cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:36,372 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:36,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:36,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:36,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b084b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:36,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:36,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:36,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:36,374 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:36,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc8d907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:36,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:36,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:36,377 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:36,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:15:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:15:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:36,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:15:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:15:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } 2024-11-28T02:15:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-28T02:15:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T02:15:36,382 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:15:36,383 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:15:36,386 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:15:36,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741860_1036 (size=84) 2024-11-28T02:15:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T02:15:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T02:15:36,795 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:15:36,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure df85333d0693eea3c91e4c0b6b737054}] 2024-11-28T02:15:36,797 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:36,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-28T02:15:36,949 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:36,950 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing df85333d0693eea3c91e4c0b6b737054 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-28T02:15:36,969 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f3/c3be998bc79a4c8d81783176ec5af5c7 is 58, key is row-test-1732760124269-3-0/f3:q1/1732760132496/Put/seqid=0 2024-11-28T02:15:36,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741861_1037 (size=5387) 2024-11-28T02:15:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T02:15:37,376 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f3/c3be998bc79a4c8d81783176ec5af5c7 2024-11-28T02:15:37,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741862_1038 (size=4469) 2024-11-28T02:15:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T02:15:37,591 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-28T02:15:37,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/mob/e81c569e82b946cdb07c47cdb39a2d86 is 59, key is row-test-1732760124269-3-0/mob:q1/1732760132510/Put/seqid=0 2024-11-28T02:15:37,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741863_1039 (size=5484) 2024-11-28T02:15:38,235 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/mob/e81c569e82b946cdb07c47cdb39a2d86 2024-11-28T02:15:38,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f3/c3be998bc79a4c8d81783176ec5af5c7 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7 2024-11-28T02:15:38,257 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7, entries=6, sequenceid=121, filesize=5.3 K 2024-11-28T02:15:38,258 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/mob/e81c569e82b946cdb07c47cdb39a2d86 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 2024-11-28T02:15:38,268 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86, entries=6, sequenceid=121, filesize=5.4 K 2024-11-28T02:15:38,269 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for df85333d0693eea3c91e4c0b6b737054 in 1319ms, sequenceid=121, compaction requested=false 2024-11-28T02:15:38,269 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for df85333d0693eea3c91e4c0b6b737054: 2024-11-28T02:15:38,269 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. for snapshot_1732760136370_default_test-1732760124269 completed. 2024-11-28T02:15:38,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.' region-info for snapshot=snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:38,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:15:38,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132] hfiles 2024-11-28T02:15:38,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 for snapshot=snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:38,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7] hfiles 2024-11-28T02:15:38,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7 for snapshot=snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:38,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86] hfiles 2024-11-28T02:15:38,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 for snapshot=snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:38,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741864_1040 (size=185) 2024-11-28T02:15:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T02:15:38,678 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:38,678 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-28T02:15:38,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-28T02:15:38,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732760136370_default_test-1732760124269 on region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:38,679 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:38,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-28T02:15:38,683 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:15:38,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure df85333d0693eea3c91e4c0b6b737054 in 1.8850 sec 2024-11-28T02:15:38,684 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:15:38,686 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-28T02:15:38,686 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-28T02:15:38,686 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:38,687 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-28T02:15:38,687 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:38,688 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-28T02:15:38,688 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:38,688 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-28T02:15:38,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741865_1041 (size=49) 2024-11-28T02:15:39,096 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:15:39,097 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:39,098 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:39,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741866_1042 (size=1146) 2024-11-28T02:15:39,511 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:15:39,521 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:15:39,522 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760136370_default_test-1732760124269 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:39,523 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:15:39,524 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-28T02:15:39,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732760136370_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } in 3.1440 sec 2024-11-28T02:15:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-28T02:15:40,554 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732760124269 completed 2024-11-28T02:15:40,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } 2024-11-28T02:15:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760140555 (current time:1732760140555). 2024-11-28T02:15:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:15:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732760140554_ns2_test-17327601242691 VERSION not specified, setting to 2 2024-11-28T02:15:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d5ce81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:40,557 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:40,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:40,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:40,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@560769c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:40,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:40,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:40,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:40,559 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:40,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7227d261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:40,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:40,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:40,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:40,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:15:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:15:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:40,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:15:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:15:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } 2024-11-28T02:15:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-28T02:15:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T02:15:40,568 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:15:40,569 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:15:40,572 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:15:40,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741867_1043 (size=86) 2024-11-28T02:15:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T02:15:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T02:15:40,980 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:15:40,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d}] 2024-11-28T02:15:40,981 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:41,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-28T02:15:41,134 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:41,134 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 30e66b01c115b81cd680f3864aaf6f9d 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-28T02:15:41,154 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/.tmp/f/1a094c37607b4bc7a971b480377e582d is 37, key is row10/f:q1/1732760126804/Put/seqid=0 2024-11-28T02:15:41,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741868_1044 (size=8413) 2024-11-28T02:15:41,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T02:15:41,560 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/.tmp/f/1a094c37607b4bc7a971b480377e582d 2024-11-28T02:15:41,570 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/.tmp/f/1a094c37607b4bc7a971b480377e582d as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d 2024-11-28T02:15:41,578 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d, entries=99, sequenceid=103, filesize=8.2 K 2024-11-28T02:15:41,580 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 30e66b01c115b81cd680f3864aaf6f9d in 445ms, sequenceid=103, compaction requested=false 2024-11-28T02:15:41,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 30e66b01c115b81cd680f3864aaf6f9d: 2024-11-28T02:15:41,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. for snapshot_1732760140554_ns2_test-17327601242691 completed. 2024-11-28T02:15:41,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d.' region-info for snapshot=snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:41,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:15:41,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d] hfiles 2024-11-28T02:15:41,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d for snapshot=snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:41,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741869_1045 (size=91) 2024-11-28T02:15:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T02:15:41,988 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:15:41,989 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-28T02:15:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-28T02:15:41,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732760140554_ns2_test-17327601242691 on region 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:41,990 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:15:41,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-28T02:15:41,994 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:15:41,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d in 1.0110 sec 2024-11-28T02:15:41,995 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:15:41,995 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:15:41,995 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:41,997 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:42,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741870_1046 (size=456) 2024-11-28T02:15:42,409 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:15:42,417 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:15:42,417 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760140554_ns2_test-17327601242691 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:42,419 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:15:42,419 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-28T02:15:42,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732760140554_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } in 1.8540 sec 2024-11-28T02:15:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-28T02:15:42,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17327601242691 completed 2024-11-28T02:15:42,724 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732760132524 2024-11-28T02:15:42,724 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-28T02:15:42,726 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-28T02:15:42,728 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732760132524_ns2:test-17327601242691 2024-11-28T02:15:42,728 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732760140554_ns2_test-17327601242691 to hdfs://localhost:38631/backupUT/backup_1732760132524/ns2/test-17327601242691/ 2024-11-28T02:15:42,733 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-28T02:15:42,769 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:38631, inputRoot=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:42,769 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_128661675_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38631/backupUT/backup_1732760132524/ns2/test-17327601242691, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38631/backupUT/backup_1732760132524/ns2/test-17327601242691/.hbase-snapshot/.tmp/snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:42,771 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-28T02:15:42,777 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760140554_ns2_test-17327601242691 to hdfs://localhost:38631/backupUT/backup_1732760132524/ns2/test-17327601242691/.hbase-snapshot/.tmp/snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:42,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741871_1047 (size=86) 2024-11-28T02:15:42,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741872_1048 (size=456) 2024-11-28T02:15:43,120 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:15:43,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:43,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:43,202 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,220 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-2738933000690721218.jar 2024-11-28T02:15:44,220 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,221 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,221 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,221 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,222 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,222 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,223 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,223 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,223 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:44,224 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:15:44,224 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:15:44,224 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:15:44,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:15:44,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:15:44,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:15:44,225 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:15:44,226 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:15:44,226 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:15:44,226 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:15:44,227 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:15:44,229 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:15:44,229 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:15:44,230 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:15:44,230 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:15:44,230 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:15:44,231 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:15:44,231 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:15:44,301 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:15:44,335 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732760140554_ns2_test-17327601242691' hfile list 2024-11-28T02:15:44,342 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-11-28T02:15:44,580 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1279409047_0001_afe6c666-8282-4b58-baa1-0e608629caa9/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-28T02:15:44,580 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1279409047_0001_afe6c666-8282-4b58-baa1-0e608629caa9/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-28T02:15:44,783 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-28T02:15:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741873_1049 (size=8413) 2024-11-28T02:15:45,293 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d output=hdfs://localhost:38631/backupUT/backup_1732760132524/ns2/test-17327601242691/archive/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d 2024-11-28T02:15:45,295 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.016M/sec 2024-11-28T02:15:45,697 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-28T02:15:45,698 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-28T02:15:45,705 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:45,705 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732760140554_ns2_test-17327601242691 finished. 2024-11-28T02:15:45,705 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732760132524_test-1732760124269 2024-11-28T02:15:45,705 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732760136370_default_test-1732760124269 to hdfs://localhost:38631/backupUT/backup_1732760132524/default/test-1732760124269/ 2024-11-28T02:15:45,705 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-28T02:15:45,736 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:38631, inputRoot=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:15:45,736 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_128661675_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38631/backupUT/backup_1732760132524/default/test-1732760124269, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38631/backupUT/backup_1732760132524/default/test-1732760124269/.hbase-snapshot/.tmp/snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:45,739 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-28T02:15:45,745 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760136370_default_test-1732760124269 to hdfs://localhost:38631/backupUT/backup_1732760132524/default/test-1732760124269/.hbase-snapshot/.tmp/snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:45,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741875_1051 (size=1146) 2024-11-28T02:15:45,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741874_1050 (size=84) 2024-11-28T02:15:46,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:46,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:46,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-7481714330460428223.jar 2024-11-28T02:15:47,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:47,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:15:47,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:15:47,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:15:47,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:15:47,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:15:47,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:15:47,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:15:47,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:15:47,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:15:47,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:15:47,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:15:47,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:15:47,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:15:47,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:15:47,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:15:47,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:15:47,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:15:47,165 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:15:47,206 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:15:47,238 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732760136370_default_test-1732760124269' hfile list 2024-11-28T02:15:47,241 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-11-28T02:15:47,337 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local689511220_0002_ed61e46b-3863-423f-85bf-ed2733a52186/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-28T02:15:47,337 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local689511220_0002_ed61e46b-3863-423f-85bf-ed2733a52186/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-28T02:15:47,416 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-28T02:15:47,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741876_1052 (size=8413) 2024-11-28T02:15:47,863 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 output=hdfs://localhost:38631/backupUT/backup_1732760132524/default/test-1732760124269/archive/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 2024-11-28T02:15:47,864 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-28T02:15:47,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741877_1053 (size=5484) 2024-11-28T02:15:48,335 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 output=hdfs://localhost:38631/backupUT/backup_1732760132524/default/test-1732760124269/archive/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 2024-11-28T02:15:48,336 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.011M/sec 2024-11-28T02:15:48,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741878_1054 (size=5387) 2024-11-28T02:15:48,805 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7 output=hdfs://localhost:38631/backupUT/backup_1732760132524/default/test-1732760124269/archive/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7 2024-11-28T02:15:48,805 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-11-28T02:15:49,385 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-28T02:15:49,387 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-28T02:15:49,395 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:49,395 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732760136370_default_test-1732760124269 finished. 2024-11-28T02:15:49,396 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17327601242691 2024-11-28T02:15:49,396 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732760124269 2024-11-28T02:15:49,446 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732760132524 2024-11-28T02:15:49,446 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-28T02:15:49,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741879_1055 (size=130) 2024-11-28T02:15:49,856 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38631/backupUT/backup_1732760132524/.backup.manifest 2024-11-28T02:15:49,858 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732760132524 finished: type=FULL,tablelist=ns2:test-17327601242691;test-1732760124269,targetRootDir=hdfs://localhost:38631/backupUT,startts=1732760136184,completets=1732760149444,bytescopied=0 2024-11-28T02:15:49,858 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-28T02:15:49,858 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:49,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732760140554_ns2_test-17327601242691" type: DISABLED 2024-11-28T02:15:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732760140554_ns2_test-17327601242691 2024-11-28T02:15:49,868 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732760140554_ns2_test-17327601242691 for backup backup_1732760132524 succeeded. 2024-11-28T02:15:49,868 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:49,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732760136370_default_test-1732760124269" type: DISABLED 2024-11-28T02:15:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732760136370_default_test-1732760124269 2024-11-28T02:15:49,872 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732760136370_default_test-1732760124269 for backup backup_1732760132524 succeeded. 2024-11-28T02:15:49,873 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-28T02:15:49,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-28T02:15:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-28T02:15:49,887 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-28T02:15:49,892 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-28T02:15:49,906 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732760132524 completed. 2024-11-28T02:15:49,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:15:49,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:15:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1732760132524', locateType=CURRENT is [region=backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., hostname=d9827d9b919f,42127,1732760115598, seqNum=2] 2024-11-28T02:15:50,087 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1732760124269 2024-11-28T02:15:50,103 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1732760124269 2024-11-28T02:15:50,251 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1732760124269 to Mob enabled CF 2024-11-28T02:15:50,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17327601242691', row='row-t20', locateType=CURRENT is [region=ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d., hostname=d9827d9b919f,42127,1732760115598, seqNum=2] 2024-11-28T02:15:50,280 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17327601242691 2024-11-28T02:15:50,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:15:50,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] assignment.SplitTableRegionProcedure(210): test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. splitKey isn't explicitly specified, will try to find a best split key from RS d9827d9b919f,42127,1732760115598 2024-11-28T02:15:50,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 because midkey is the same as first or last row 2024-11-28T02:15:50,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7 because midkey is the same as first or last row 2024-11-28T02:15:50,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 because midkey is the same as first or last row 2024-11-28T02:15:50,291 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.HRegion(2902): Flushing df85333d0693eea3c91e4c0b6b737054 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-28T02:15:50,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f/6f13f53cc814494699abef649aa85eca is 59, key is row-test-1732760124269-1-10/f:q1/1732760149944/Put/seqid=0 2024-11-28T02:15:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741880_1056 (size=10796) 2024-11-28T02:15:50,716 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f/6f13f53cc814494699abef649aa85eca 2024-11-28T02:15:50,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054 is 63, key is row-test-1732760124269-3-100/mob:q1/1732760150236/Put/seqid=0 2024-11-28T02:15:50,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741881_1057 (size=5591) 2024-11-28T02:15:51,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:15:51,145 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/mobdir/data/default/test-1732760124269/33f2a14eb25263e694b97d0d2bc810e0/mob/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:15:51,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/mob/d27708e4bf90413f8d92cd5dddf8a1b4, store: [table=test-1732760124269 family=mob region=df85333d0693eea3c91e4c0b6b737054] 2024-11-28T02:15:51,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/mob/d27708e4bf90413f8d92cd5dddf8a1b4 is 192, key is row-test-1732760124269-3-100/mob:q1/1732760150236/Put/seqid=0 2024-11-28T02:15:51,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741882_1058 (size=13760) 2024-11-28T02:15:51,553 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/mob/d27708e4bf90413f8d92cd5dddf8a1b4 2024-11-28T02:15:51,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f/6f13f53cc814494699abef649aa85eca as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/6f13f53cc814494699abef649aa85eca 2024-11-28T02:15:51,567 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/6f13f53cc814494699abef649aa85eca, entries=99, sequenceid=334, filesize=10.5 K 2024-11-28T02:15:51,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/mob/d27708e4bf90413f8d92cd5dddf8a1b4 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/d27708e4bf90413f8d92cd5dddf8a1b4 2024-11-28T02:15:51,574 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/d27708e4bf90413f8d92cd5dddf8a1b4, entries=111, sequenceid=334, filesize=13.4 K 2024-11-28T02:15:51,575 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for df85333d0693eea3c91e4c0b6b737054 in 1284ms, sequenceid=334, compaction requested=false 2024-11-28T02:15:51,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.HRegion(2603): Flush status journal for df85333d0693eea3c91e4c0b6b737054: 2024-11-28T02:15:51,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/6f13f53cc814494699abef649aa85eca because midkey is the same as first or last row 2024-11-28T02:15:51,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7 because midkey is the same as first or last row 2024-11-28T02:15:51,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/d27708e4bf90413f8d92cd5dddf8a1b4 because midkey is the same as first or last row 2024-11-28T02:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=d9827d9b919f,42127,1732760115598 2024-11-28T02:15:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-28T02:15:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.CallRunner(138): callId: 299 service: MasterService methodName: SplitRegion size: 99 connection: 172.17.0.2:41218 deadline: 1732760210286, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-28T02:15:51,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1732760124269 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:15:51,583 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:15:51,584 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T02:15:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732760124269,, stopping at row=test-1732760124269 ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,596 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732760124269,, stopping at row=test-1732760124269 ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,603 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1322 2024-11-28T02:15:51,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,641 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-28T02:15:51,645 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732760151603 2024-11-28T02:15:51,653 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732760132524} 2024-11-28T02:15:51,653 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-28T02:15:51,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:15:51,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760151699 (current time:1732760151699). 2024-11-28T02:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:15:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-28T02:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3bd40d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:51,702 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:51,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:51,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:51,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe14d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:51,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:51,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:51,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:51,705 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:51,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5991db1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:51,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:51,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:51,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:51,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:15:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:15:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:51,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:15:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:15:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:15:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-28T02:15:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-28T02:15:51,713 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:15:51,714 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:15:51,717 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:15:51,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741883_1059 (size=52) 2024-11-28T02:15:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-28T02:15:52,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-28T02:15:52,124 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:15:52,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a}] 2024-11-28T02:15:52,125 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:52,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-28T02:15:52,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:52,277 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing a2d65c385b1b9e1c5b04dfa38326a47a 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-28T02:15:52,294 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/f87bf723c61349d4b13e444028724d17 is 158, key is trslm:hdfs://localhost:38631/backupUT\x00ns2:test-17327601242691/meta:log-roll-map/1732760149433/Put/seqid=0 2024-11-28T02:15:52,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741884_1060 (size=5914) 2024-11-28T02:15:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-28T02:15:52,699 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/f87bf723c61349d4b13e444028724d17 2024-11-28T02:15:52,710 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/5e26820e292547718ba9895fc2aefe8a is 433, key is session:backup_1732760132524/session:context/1732760149891/Put/seqid=0 2024-11-28T02:15:52,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741885_1061 (size=5535) 2024-11-28T02:15:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-28T02:15:53,115 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/5e26820e292547718ba9895fc2aefe8a 2024-11-28T02:15:53,122 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/f87bf723c61349d4b13e444028724d17 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17 2024-11-28T02:15:53,129 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17, entries=6, sequenceid=17, filesize=5.8 K 2024-11-28T02:15:53,130 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/5e26820e292547718ba9895fc2aefe8a as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a 2024-11-28T02:15:53,135 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a, entries=2, sequenceid=17, filesize=5.4 K 2024-11-28T02:15:53,137 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for a2d65c385b1b9e1c5b04dfa38326a47a in 859ms, sequenceid=17, compaction requested=false 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for a2d65c385b1b9e1c5b04dfa38326a47a: 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. for snapshot_backup_system completed. 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.' region-info for snapshot=snapshot_backup_system 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17] hfiles 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17 for snapshot=snapshot_backup_system 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a] hfiles 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba for snapshot=snapshot_backup_system 2024-11-28T02:15:53,137 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a for snapshot=snapshot_backup_system 2024-11-28T02:15:53,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741886_1062 (size=173) 2024-11-28T02:15:53,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:15:53,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-28T02:15:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-28T02:15:53,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:53,544 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:15:53,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-28T02:15:53,547 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:15:53,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a in 1.4210 sec 2024-11-28T02:15:53,548 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:15:53,549 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:15:53,549 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-28T02:15:53,550 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-28T02:15:53,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741887_1063 (size=810) 2024-11-28T02:15:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-28T02:15:53,961 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:15:53,969 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:15:53,969 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_backup_system 2024-11-28T02:15:53,971 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:15:53,971 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-28T02:15:53,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2610 sec 2024-11-28T02:15:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-28T02:15:55,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-28T02:15:55,884 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732760151603 started at 1732760155884. 2024-11-28T02:15:55,886 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732760151603 has been started. 2024-11-28T02:15:55,891 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17327601242691, test-1732760124269] 2024-11-28T02:15:55,895 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732760125591for backupID backup_1732760151603 2024-11-28T02:15:55,895 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-28T02:15:55,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-28T02:15:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-28T02:15:55,896 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-28T02:15:55,896 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-28T02:15:55,896 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-28T02:15:55,896 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-28T02:15:55,897 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:55,897 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:55,899 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-28T02:15:55,899 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-28T02:15:55,899 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:15:55,900 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,900 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-28T02:15:55,900 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:55,900 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:55,901 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-28T02:15:55,901 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:55,901 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-28T02:15:55,901 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-28T02:15:55,901 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-28T02:15:55,901 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-28T02:15:55,901 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-28T02:15:55,901 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-28T02:15:55,902 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-28T02:15:55,902 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'd9827d9b919f,42127,1732760115598' joining acquired barrier for procedure (rolllog) in zk 2024-11-28T02:15:55,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,904 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:55,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:15:55,904 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:15:55,904 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-28T02:15:55,904 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-28T02:15:55,905 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:15:55,905 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:15:55,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:55,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,906 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:15:55,907 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'd9827d9b919f,42127,1732760115598' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-28T02:15:55,907 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@58acb298[Count = 0] remaining members to acquire global barrier 2024-11-28T02:15:55,907 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-28T02:15:55,907 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:55,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:55,909 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:55,909 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:55,910 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-28T02:15:55,910 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,910 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-28T02:15:55,910 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-28T02:15:55,910 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,910 INFO [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732760136284 highest: 1732760136284 on d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,910 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.meta:.meta(num 1732760136242) roll requested 2024-11-28T02:15:55,932 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760155910.meta, exclude list is [], retry=0 2024-11-28T02:15:55,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:55,938 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760155910.meta 2024-11-28T02:15:55,938 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:55,938 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta is not closed yet, will try archiving it next time 2024-11-28T02:15:55,938 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1:(num 1732760136264) roll requested 2024-11-28T02:15:55,940 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta not finished, retry = 0 2024-11-28T02:15:55,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741857_1033 (size=93) 2024-11-28T02:15:55,953 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760155938, exclude list is [], retry=0 2024-11-28T02:15:55,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:55,957 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760155938 2024-11-28T02:15:55,957 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:55,957 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 is not closed yet, will try archiving it next time 2024-11-28T02:15:55,957 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0:(num 1732760136284) roll requested 2024-11-28T02:15:55,958 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760125591 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760125591 2024-11-28T02:15:55,959 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 not finished, retry = 0 2024-11-28T02:15:55,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741858_1034 (size=30490) 2024-11-28T02:15:55,971 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760155958, exclude list is [], retry=0 2024-11-28T02:15:55,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:15:55,975 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760155958 2024-11-28T02:15:55,975 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:15:55,975 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 is not closed yet, will try archiving it next time 2024-11-28T02:15:55,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741859_1035 (size=1198) 2024-11-28T02:15:55,977 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 not finished, retry = 0 2024-11-28T02:15:55,991 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 81 2024-11-28T02:15:55,991 INFO [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732760155958 on d9827d9b919f,42127,1732760115598 2024-11-28T02:15:55,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:15:56,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:15:56,008 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-28T02:15:56,009 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'd9827d9b919f,42127,1732760115598' in zk 2024-11-28T02:15:56,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,011 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-28T02:15:56,011 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-28T02:15:56,011 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,011 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:15:56,011 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-28T02:15:56,011 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:15:56,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:15:56,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:15:56,012 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:56,013 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,013 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:15:56,014 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:56,014 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,015 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'd9827d9b919f,42127,1732760115598': 2024-11-28T02:15:56,015 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-28T02:15:56,015 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'd9827d9b919f,42127,1732760115598' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-28T02:15:56,015 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-28T02:15:56,015 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-28T02:15:56,015 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-28T02:15:56,015 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-28T02:15:56,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:15:56,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:15:56,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-28T02:15:56,018 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-28T02:15:56,018 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:56,018 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:15:56,018 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:56,018 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:15:56,018 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-28T02:15:56,018 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:15:56,018 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:15:56,019 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,019 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-28T02:15:56,019 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:56,020 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:15:56,020 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:56,021 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:15:56,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:15:56,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-28T02:15:56,024 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-28T02:15:56,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:15:56,024 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-28T02:15:56,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:15:56,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-28T02:15:56,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-28T02:15:56,025 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-11-28T02:15:56,025 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-28T02:15:56,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:15:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-28T02:15:56,036 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {d9827d9b919f:42127=1732760125591} newestTimestamps: {d9827d9b919f:42127=1732760136284} 2024-11-28T02:15:56,039 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: d9827d9b919f%2C42127%2C1732760115598.meta.1732760119169.meta 2024-11-28T02:15:56,039 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta 2024-11-28T02:15:56,039 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: d9827d9b919f%2C42127%2C1732760115598.meta.1732760155910.meta 2024-11-28T02:15:56,039 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 2024-11-28T02:15:56,039 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760155958 2024-11-28T02:15:56,039 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 2024-11-28T02:15:56,039 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760155938 2024-11-28T02:15:56,041 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta 2024-11-28T02:15:56,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741891_1067 (size=364) 2024-11-28T02:15:56,453 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17327601242691 target: hdfs://localhost:38631/backupUT/backup_1732760151603/ns2/test-17327601242691 descriptor: 'ns2:test-17327601242691', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:56,453 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-28T02:15:56,456 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17327601242691 2024-11-28T02:15:56,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741892_1068 (size=50) 2024-11-28T02:15:56,862 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17327601242691 2024-11-28T02:15:56,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741893_1069 (size=908) 2024-11-28T02:15:57,275 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732760124269 target: hdfs://localhost:38631/backupUT/backup_1732760151603/default/test-1732760124269 descriptor: 'test-1732760124269', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:15:57,276 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-28T02:15:57,278 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732760124269 2024-11-28T02:15:57,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741894_1070 (size=53) 2024-11-28T02:15:57,685 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732760124269 2024-11-28T02:15:57,694 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:38631/backupUT/.tmp/backup_1732760151603 from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284;hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 2024-11-28T02:15:57,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d30f64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:57,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:15:57,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:15:57,697 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:15:57,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:15:57,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:15:57,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7dffe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:57,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:15:57,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:15:57,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:57,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:15:57,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685fa6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:15:57,700 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:15:57,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is d9827d9b919f,43413,1732760114080 2024-11-28T02:15:57,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3644b00a 2024-11-28T02:15:57,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:15:57,702 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:15:57,707 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-28T02:15:57,708 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17327601242691 2024-11-28T02:15:57,708 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17327601242691,, stopping at row=ns2:test-17327601242691 ,, for max=2147483647 with caching=100 2024-11-28T02:15:57,708 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:15:57,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:15:57,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:15:57,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:15:57,712 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17327601242691: ns2:test-17327601242691; 2024-11-28T02:15:57,713 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732760124269 2024-11-28T02:15:57,713 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732760124269,, stopping at row=test-1732760124269 ,, for max=2147483647 with caching=100 2024-11-28T02:15:57,715 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732760124269: test-1732760124269; 2024-11-28T02:15:57,715 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-28T02:15:57,715 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/hbase-staging/partitions_47afd46f-ed3f-478b-8666-e2a6ec1c9e29 2024-11-28T02:15:57,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741895_1071 (size=192) 2024-11-28T02:15:58,140 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:58,140 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:58,141 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-16097841140405107952.jar 2024-11-28T02:15:59,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,126 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,127 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,128 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:15:59,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:15:59,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:15:59,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:15:59,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:15:59,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:15:59,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:15:59,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:15:59,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:15:59,131 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:15:59,131 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:15:59,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:15:59,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:15:59,134 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17327601242691,default:test-1732760124269 2024-11-28T02:15:59,134 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:15:59,134 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:15:59,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:59,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:15:59,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:15:59,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:15:59,424 DEBUG [master/d9827d9b919f:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732760124269 2024-11-28T02:15:59,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741896_1072 (size=950) 2024-11-28T02:15:59,838 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/.tabledesc/.tableinfo.0000000002.908 2024-11-28T02:15:59,838 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/.tabledesc/.tableinfo.0000000003.950 2024-11-28T02:15:59,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732760124269 in 418 msec 2024-11-28T02:16:00,120 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-18195534421324153873.jar 2024-11-28T02:16:00,134 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:16:00,160 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 for WAL files 2024-11-28T02:16:00,161 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 2024-11-28T02:16:00,162 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 for WAL files 2024-11-28T02:16:00,163 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 2024-11-28T02:16:00,261 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1592318737_0003_425cafdf-3b55-4e35-9d03-7b8dec67696a/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-28T02:16:00,261 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1592318737_0003_425cafdf-3b55-4e35-9d03-7b8dec67696a/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-28T02:16:00,361 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 for hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 (-9223372036854775808:9223372036854775807) length:30490 2024-11-28T02:16:00,383 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-28T02:16:00,384 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-28T02:16:00,438 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 for hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 (-9223372036854775808:9223372036854775807) length:1198 2024-11-28T02:16:00,442 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-28T02:16:00,443 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-28T02:16:00,542 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c292359, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:00,542 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:00,542 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:00,543 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:00,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:00,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:00,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5089c34d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:00,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:00,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:00,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:00,545 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:00,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2684fe5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:00,547 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:00,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:00,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:00,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:00,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17327601242691', row='row-t20', locateType=CURRENT is [region=ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d., hostname=d9827d9b919f,42127,1732760115598, seqNum=2] 2024-11-28T02:16:00,552 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-28T02:16:00,552 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:00,552 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:00,552 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:00,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:00,552 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-28T02:16:00,552 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: d9827d9b919f 2024-11-28T02:16:00,557 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/backupUT/.tmp/backup_1732760151603/_temporary/0/_temporary/attempt_local1592318737_0003_r_000000_0/ns2/test-17327601242691/f/19451fecb5644753922aba5aeaa525bb is 38, key is row-t20/f:q1/1732760150269/Put/seqid=0 2024-11-28T02:16:00,559 WARN [Thread-948 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [d9827d9b919f:42127] Specified favored nodes: [d9827d9b919f:42127] 2024-11-28T02:16:00,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741897_1073 (size=5275) 2024-11-28T02:16:00,990 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@748bf8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:00,990 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:00,990 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:00,991 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:00,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:00,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:00,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@191cc8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:00,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:00,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:00,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:00,992 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:00,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459a4add, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:00,993 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:00,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:00,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:00,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:00,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732760124269', row='row-test-1732760124269-1-0', locateType=CURRENT is [region=test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., hostname=d9827d9b919f,42127,1732760115598, seqNum=107] 2024-11-28T02:16:00,997 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-28T02:16:00,997 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:00,997 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:00,998 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:00,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:00,998 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732760124269-1-0] 2024-11-28T02:16:00,998 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: d9827d9b919f 2024-11-28T02:16:01,008 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21558ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:01,008 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:01,008 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:01,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:01,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:01,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:01,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b13c860, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:01,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:01,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:01,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:01,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:01,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@413557b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:01,012 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:01,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:01,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:01,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:01,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732760124269', row='row-test-1732760124269-3-0', locateType=CURRENT is [region=test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., hostname=d9827d9b919f,42127,1732760115598, seqNum=107] 2024-11-28T02:16:01,016 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-28T02:16:01,016 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:01,016 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:01,016 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:01,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:01,016 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732760124269-3-0] 2024-11-28T02:16:01,016 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: d9827d9b919f 2024-11-28T02:16:01,026 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/backupUT/.tmp/backup_1732760151603/_temporary/0/_temporary/attempt_local1592318737_0003_r_000001_0/default/test-1732760124269/f/4cecf22bb4b34b1295ca307e0fdc8c9b is 59, key is row-test-1732760124269-1-10/f:q1/1732760149944/Put/seqid=0 2024-11-28T02:16:01,028 WARN [Thread-960 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [d9827d9b919f:42127] Specified favored nodes: [d9827d9b919f:42127] 2024-11-28T02:16:01,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741898_1074 (size=10770) 2024-11-28T02:16:01,432 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/backupUT/.tmp/backup_1732760151603/_temporary/0/_temporary/attempt_local1592318737_0003_r_000001_0/default/test-1732760124269/mob/6bb2f4b24f2e4ebcaf6170b02e252794 is 63, key is row-test-1732760124269-3-100/mob:q1/1732760150236/Put/seqid=0 2024-11-28T02:16:01,434 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [d9827d9b919f:42127] Specified favored nodes: [d9827d9b919f:42127] 2024-11-28T02:16:01,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741899_1075 (size=11811) 2024-11-28T02:16:02,304 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:38631/backupUT 2024-11-28T02:16:02,304 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732760151603 2024-11-28T02:16:02,304 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-28T02:16:02,324 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:38631/backupUT/.tmp/backup_1732760151603, hdfs://localhost:38631/backupUT] 2024-11-28T02:16:02,399 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:16:02,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741900_1076 (size=10770) 2024-11-28T02:16:02,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741901_1077 (size=5275) 2024-11-28T02:16:03,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741902_1078 (size=11811) 2024-11-28T02:16:04,025 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-28T02:16:04,028 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732760151603 2024-11-28T02:16:04,028 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-11-28T02:16:04,028 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1957778865_0004 completed: true true 2024-11-28T02:16:04,029 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3825260 FILE: Number of bytes written=6211346 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231425 HDFS: Number of bytes written=2433143 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=249 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=340 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-11-28T02:16:04,029 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:38631/backupUT/.tmp/backup_1732760151603 to hdfs://localhost:38631/backupUT finished. 2024-11-28T02:16:04,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:ns2:test-17327601242691:', locateType=CURRENT is [region=backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d., hostname=d9827d9b919f,42127,1732760115598, seqNum=2] 2024-11-28T02:16:04,044 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732760151603 2024-11-28T02:16:04,047 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732760132524} 2024-11-28T02:16:04,047 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-28T02:16:04,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741903_1079 (size=384) 2024-11-28T02:16:04,454 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38631/backupUT/backup_1732760151603/.backup.manifest 2024-11-28T02:16:04,454 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732760151603 finished: type=INCREMENTAL,tablelist=ns2:test-17327601242691;test-1732760124269,targetRootDir=hdfs://localhost:38631/backupUT,startts=1732760155884,completets=1732760164044,bytescopied=0 2024-11-28T02:16:04,454 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-28T02:16:04,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-28T02:16:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-28T02:16:04,462 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-28T02:16:04,464 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-28T02:16:04,467 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732760151603 completed. 2024-11-28T02:16:04,467 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-28T02:16:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:04,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:04,482 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38631/backupUT/backup_1732760151603/.backup.manifest 2024-11-28T02:16:04,485 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732760151603/.backup.manifest 2024-11-28T02:16:04,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732760124269 from 'test-1732760124269', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732760124269', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732760124269 2024-11-28T02:16:04,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741904_1080 (size=908) 2024-11-28T02:16:04,898 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/.tabledesc/.tableinfo.0000000003.950 2024-11-28T02:16:04,898 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/.tabledesc/.tableinfo.0000000004.908 2024-11-28T02:16:04,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732760124269}] 2024-11-28T02:16:04,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE}] 2024-11-28T02:16:04,904 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE 2024-11-28T02:16:04,905 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=CLOSING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:16:04,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE because future has completed 2024-11-28T02:16:04,907 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:16:04,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:16:05,060 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,060 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:16:05,060 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing df85333d0693eea3c91e4c0b6b737054, disabling compactions & flushes 2024-11-28T02:16:05,060 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:05,060 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:05,060 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. after waiting 0 ms 2024-11-28T02:16:05,060 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:05,065 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-28T02:16:05,066 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:05,066 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:05,066 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for df85333d0693eea3c91e4c0b6b737054: Waiting for close lock at 1732760165060Running coprocessor pre-close hooks at 1732760165060Disabling compacts and flushes for region at 1732760165060Disabling writes for close at 1732760165060Writing region close event to WAL at 1732760165062 (+2 ms)Running coprocessor post-close hooks at 1732760165066 (+4 ms)Closed at 1732760165066 2024-11-28T02:16:05,066 WARN [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: df85333d0693eea3c91e4c0b6b737054 to self. 2024-11-28T02:16:05,068 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,069 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=CLOSED 2024-11-28T02:16:05,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:16:05,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-28T02:16:05,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 in 164 msec 2024-11-28T02:16:05,074 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE; state=CLOSED, location=d9827d9b919f,42127,1732760115598; forceNewPlan=false, retain=true 2024-11-28T02:16:05,224 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:16:05,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE because future has completed 2024-11-28T02:16:05,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:16:05,383 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:05,383 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => df85333d0693eea3c91e4c0b6b737054, NAME => 'test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:16:05,383 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:16:05,384 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732760124269 df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,384 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:16:05,384 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,384 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,386 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,387 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df85333d0693eea3c91e4c0b6b737054 columnFamilyName f 2024-11-28T02:16:05,387 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:05,394 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 2024-11-28T02:16:05,398 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/6f13f53cc814494699abef649aa85eca 2024-11-28T02:16:05,398 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(327): Store=df85333d0693eea3c91e4c0b6b737054/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:16:05,398 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,399 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df85333d0693eea3c91e4c0b6b737054 columnFamilyName f2 2024-11-28T02:16:05,400 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:05,400 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(327): Store=df85333d0693eea3c91e4c0b6b737054/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:16:05,400 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,401 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df85333d0693eea3c91e4c0b6b737054 columnFamilyName mob 2024-11-28T02:16:05,402 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:05,408 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/d27708e4bf90413f8d92cd5dddf8a1b4 2024-11-28T02:16:05,412 DEBUG [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 2024-11-28T02:16:05,412 INFO [StoreOpener-df85333d0693eea3c91e4c0b6b737054-1 {}] regionserver.HStore(327): Store=df85333d0693eea3c91e4c0b6b737054/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:16:05,413 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,413 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,415 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,416 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,416 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,416 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732760124269 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-28T02:16:05,418 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,419 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened df85333d0693eea3c91e4c0b6b737054; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10471788320, jitterRate=-0.02473871409893036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-28T02:16:05,419 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:05,421 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for df85333d0693eea3c91e4c0b6b737054: Running coprocessor pre-open hook at 1732760165384Writing region info on filesystem at 1732760165384Initializing all the Stores at 1732760165385 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760165385Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760165385Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760165385Cleaning up temporary data from old regions at 1732760165416 (+31 ms)Running coprocessor post-open hooks at 1732760165419 (+3 ms)Region opened successfully at 1732760165421 (+2 ms) 2024-11-28T02:16:05,422 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., pid=46, masterSystemTime=1732760165379 2024-11-28T02:16:05,424 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:05,425 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:05,425 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=df85333d0693eea3c91e4c0b6b737054, regionState=OPEN, openSeqNum=338, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:16:05,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:16:05,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-28T02:16:05,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure df85333d0693eea3c91e4c0b6b737054, server=d9827d9b919f,42127,1732760115598 in 201 msec 2024-11-28T02:16:05,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-28T02:16:05,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732760124269, region=df85333d0693eea3c91e4c0b6b737054, REOPEN/MOVE in 527 msec 2024-11-28T02:16:05,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-28T02:16:05,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732760124269 in 531 msec 2024-11-28T02:16:05,437 DEBUG [PEWorker-5 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1732760124269 2024-11-28T02:16:05,443 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f3/c3be998bc79a4c8d81783176ec5af5c7 2024-11-28T02:16:05,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732760124269 in 958 msec 2024-11-28T02:16:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T02:16:05,447 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732760124269 completed 2024-11-28T02:16:05,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,478 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-28T02:16:05,482 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732760165447 2024-11-28T02:16:05,484 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1732760151603} 2024-11-28T02:16:05,484 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732760132524} 2024-11-28T02:16:05,484 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-28T02:16:05,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,525 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1732760124269 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1732760124269 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:16:05,529 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732760165447,startts=0,failedts=1732760165526,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1732760124269 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-28T02:16:05,529 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-28T02:16:05,531 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-28T02:16:05,531 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-28T02:16:05,533 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-28T02:16:05,533 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-28T02:16:05,535 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-28T02:16:05,538 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732760165447 failed. 2024-11-28T02:16:05,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:05,556 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1732760124269, ns2:test-17327601242691, ns3:test-17327601242692, ns4:test-17327601242693] 2024-11-28T02:16:05,556 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-28T02:16:05,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:16:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760165560 (current time:1732760165560). 2024-11-28T02:16:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:16:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-28T02:16:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315da5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:05,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:05,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:05,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:05,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff140ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:05,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:05,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:05,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:05,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:05,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721fef02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:05,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:05,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:05,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:05,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:16:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:16:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:05,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:16:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:16:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-28T02:16:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-28T02:16:05,570 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:16:05,571 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:16:05,573 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:16:05,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741905_1081 (size=52) 2024-11-28T02:16:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-28T02:16:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-28T02:16:05,979 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:16:05,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a}] 2024-11-28T02:16:05,981 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:06,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-11-28T02:16:06,133 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:06,133 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing a2d65c385b1b9e1c5b04dfa38326a47a 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-28T02:16:06,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/c98da890d50746ecbbb10200c71bf935 is 158, key is trslm:hdfs://localhost:38631/backupUT\x00ns2:test-17327601242691/meta:log-roll-map/1732760164034/Put/seqid=0 2024-11-28T02:16:06,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741906_1082 (size=5745) 2024-11-28T02:16:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-28T02:16:06,544 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/c98da890d50746ecbbb10200c71bf935 2024-11-28T02:16:06,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/f4071ad0b88840998cb64d2c91a76486 is 1245, key is session:backup_1732760165447/session:context/1732760165534/Put/seqid=0 2024-11-28T02:16:06,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741907_1083 (size=6682) 2024-11-28T02:16:06,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-28T02:16:06,958 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/f4071ad0b88840998cb64d2c91a76486 2024-11-28T02:16:06,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/c98da890d50746ecbbb10200c71bf935 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935 2024-11-28T02:16:06,971 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935, entries=4, sequenceid=31, filesize=5.6 K 2024-11-28T02:16:06,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/f4071ad0b88840998cb64d2c91a76486 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486 2024-11-28T02:16:06,979 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486, entries=3, sequenceid=31, filesize=6.5 K 2024-11-28T02:16:06,980 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for a2d65c385b1b9e1c5b04dfa38326a47a in 847ms, sequenceid=31, compaction requested=true 2024-11-28T02:16:06,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for a2d65c385b1b9e1c5b04dfa38326a47a: 2024-11-28T02:16:06,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. for snapshot_backup_system completed. 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.' region-info for snapshot=snapshot_backup_system 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935] hfiles 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17 for snapshot=snapshot_backup_system 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935 for snapshot=snapshot_backup_system 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486] hfiles 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba for snapshot=snapshot_backup_system 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a for snapshot=snapshot_backup_system 2024-11-28T02:16:06,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486 for snapshot=snapshot_backup_system 2024-11-28T02:16:06,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741908_1084 (size=251) 2024-11-28T02:16:07,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:07,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-28T02:16:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-11-28T02:16:07,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:07,388 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:07,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-28T02:16:07,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a in 1.4100 sec 2024-11-28T02:16:07,391 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:16:07,392 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:16:07,392 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:16:07,393 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-28T02:16:07,394 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-28T02:16:07,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741909_1085 (size=888) 2024-11-28T02:16:07,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-28T02:16:07,804 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:16:07,812 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:16:07,813 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_backup_system 2024-11-28T02:16:07,814 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:16:07,814 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-28T02:16:07,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2460 sec 2024-11-28T02:16:09,420 DEBUG [master/d9827d9b919f:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732760124269 2024-11-28T02:16:09,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741910_1086 (size=950) 2024-11-28T02:16:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-28T02:16:09,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-28T02:16:09,724 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732760165539 started at 1732760169724. 2024-11-28T02:16:09,726 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732760165539 has been started. 2024-11-28T02:16:09,730 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-28T02:16:09,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-28T02:16:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-28T02:16:09,731 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-28T02:16:09,731 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-28T02:16:09,731 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-28T02:16:09,731 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-28T02:16:09,732 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,732 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:09,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-28T02:16:09,734 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,734 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-28T02:16:09,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:16:09,734 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,734 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-28T02:16:09,735 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:09,735 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-28T02:16:09,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:09,736 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-28T02:16:09,736 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-28T02:16:09,736 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-28T02:16:09,736 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-28T02:16:09,736 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-28T02:16:09,736 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-28T02:16:09,736 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-28T02:16:09,736 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'd9827d9b919f,42127,1732760115598' joining acquired barrier for procedure (rolllog) in zk 2024-11-28T02:16:09,738 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:09,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:16:09,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:16:09,738 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-28T02:16:09,739 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-28T02:16:09,739 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:16:09,739 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:16:09,740 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:09,740 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,741 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:16:09,741 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'd9827d9b919f,42127,1732760115598' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-28T02:16:09,741 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1041b422[Count = 0] remaining members to acquire global barrier 2024-11-28T02:16:09,741 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-28T02:16:09,741 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:09,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:09,744 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:09,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:09,744 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-28T02:16:09,744 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,744 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-28T02:16:09,744 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-28T02:16:09,744 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,744 INFO [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732760155958 highest: 1732760155958 on d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,744 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.meta:.meta(num 1732760155910) roll requested 2024-11-28T02:16:09,758 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760169745.meta, exclude list is [], retry=0 2024-11-28T02:16:09,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:16:09,762 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760155910.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760169745.meta 2024-11-28T02:16:09,762 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:16:09,762 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760155910.meta is not closed yet, will try archiving it next time 2024-11-28T02:16:09,762 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1:(num 1732760155938) roll requested 2024-11-28T02:16:09,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741888_1064 (size=1903) 2024-11-28T02:16:09,764 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760155910.meta not finished, retry = 0 2024-11-28T02:16:09,776 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760169762, exclude list is [], retry=0 2024-11-28T02:16:09,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:16:09,779 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760155938 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760169762 2024-11-28T02:16:09,779 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:16:09,779 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760155938 is not closed yet, will try archiving it next time 2024-11-28T02:16:09,779 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0:(num 1732760155958) roll requested 2024-11-28T02:16:09,779 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760136264 2024-11-28T02:16:09,780 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760155938 not finished, retry = 0 2024-11-28T02:16:09,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741889_1065 (size=4927) 2024-11-28T02:16:09,792 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779, exclude list is [], retry=0 2024-11-28T02:16:09,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:16:09,795 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760155958 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 2024-11-28T02:16:09,795 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:16:09,795 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760155958 is not closed yet, will try archiving it next time 2024-11-28T02:16:09,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741890_1066 (size=93) 2024-11-28T02:16:09,797 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760155958 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760155958 2024-11-28T02:16:09,805 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-28T02:16:09,805 INFO [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732760169779 on d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:09,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:09,821 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-28T02:16:09,821 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'd9827d9b919f,42127,1732760115598' in zk 2024-11-28T02:16:09,823 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-28T02:16:09,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,823 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-28T02:16:09,823 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-28T02:16:09,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:16:09,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:16:09,824 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:16:09,824 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:16:09,824 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:09,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:16:09,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:09,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'd9827d9b919f,42127,1732760115598': 2024-11-28T02:16:09,827 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'd9827d9b919f,42127,1732760115598' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-28T02:16:09,827 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-28T02:16:09,827 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-28T02:16:09,827 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-28T02:16:09,827 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-28T02:16:09,827 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-28T02:16:09,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-28T02:16:09,829 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:16:09,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:16:09,830 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:16:09,830 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-28T02:16:09,830 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:16:09,830 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,830 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:09,830 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,831 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:16:09,831 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:09,832 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,832 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/.tabledesc/.tableinfo.0000000004.908 2024-11-28T02:16:09,832 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/.tabledesc/.tableinfo.0000000005.950 2024-11-28T02:16:09,832 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,832 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:16:09,832 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:09,833 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732760124269 in 413 msec 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:09,835 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:09,835 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:09,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:16:09,835 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-28T02:16:09,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-28T02:16:09,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-28T02:16:09,836 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-28T02:16:09,836 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:16:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-28T02:16:09,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-28T02:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760169847 (current time:1732760169847). 2024-11-28T02:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732760169846_backup_system_bulk VERSION not specified, setting to 2 2024-11-28T02:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b379887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:09,848 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:09,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:09,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:09,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce6abe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:09,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:09,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:09,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:09,850 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:09,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14868327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:09,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:09,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:09,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:09,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:16:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:16:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:09,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:16:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-28T02:16:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-28T02:16:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:16:09,856 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:16:09,857 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:16:09,859 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:16:09,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741914_1090 (size=76) 2024-11-28T02:16:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:16:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:16:10,265 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:16:10,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63374509f749d336450706661d23059d}] 2024-11-28T02:16:10,266 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63374509f749d336450706661d23059d 2024-11-28T02:16:10,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-28T02:16:10,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:16:10,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 63374509f749d336450706661d23059d: 2024-11-28T02:16:10,418 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. for snapshot_1732760169846_backup_system_bulk completed. 2024-11-28T02:16:10,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d.' region-info for snapshot=snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:10,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:16:10,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-28T02:16:10,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-28T02:16:10,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741915_1091 (size=61) 2024-11-28T02:16:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:16:10,595 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-28T02:16:10,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:16:10,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-28T02:16:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-28T02:16:10,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732760169846_backup_system_bulk on region 63374509f749d336450706661d23059d 2024-11-28T02:16:10,826 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 63374509f749d336450706661d23059d 2024-11-28T02:16:10,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-28T02:16:10,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 63374509f749d336450706661d23059d in 562 msec 2024-11-28T02:16:10,829 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:16:10,830 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:16:10,830 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:16:10,830 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:10,831 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:10,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741916_1092 (size=702) 2024-11-28T02:16:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:16:11,242 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:16:11,247 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:16:11,247 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760169846_backup_system_bulk to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:11,248 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:16:11,249 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-28T02:16:11,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732760169846_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.3950 sec 2024-11-28T02:16:11,501 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 30e66b01c115b81cd680f3864aaf6f9d, had cached 0 bytes from a total of 8413 2024-11-28T02:16:12,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T02:16:12,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-28T02:16:12,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } 2024-11-28T02:16:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760172004 (current time:1732760172004). 2024-11-28T02:16:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:16:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732760172004_ns3_test-17327601242692 VERSION not specified, setting to 2 2024-11-28T02:16:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55950022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:12,006 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:12,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:12,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:12,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41abcd22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:12,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:12,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:12,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:12,008 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:12,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a114fb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:12,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:12,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:12,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:12,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:16:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:16:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:12,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:12,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:16:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } 2024-11-28T02:16:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-28T02:16:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:16:12,014 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:16:12,014 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:16:12,017 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:16:12,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741917_1093 (size=86) 2024-11-28T02:16:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:16:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:16:12,423 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:16:12,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 795ba51d3cf9aa45e08a6768f6136859}] 2024-11-28T02:16:12,424 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:16:12,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-28T02:16:12,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:16:12,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 795ba51d3cf9aa45e08a6768f6136859: 2024-11-28T02:16:12,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. for snapshot_1732760172004_ns3_test-17327601242692 completed. 2024-11-28T02:16:12,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859.' region-info for snapshot=snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:12,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:16:12,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-28T02:16:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741918_1094 (size=52) 2024-11-28T02:16:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:16:12,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:16:12,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-28T02:16:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-28T02:16:12,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732760172004_ns3_test-17327601242692 on region 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:16:12,984 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:16:12,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-28T02:16:12,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 795ba51d3cf9aa45e08a6768f6136859 in 562 msec 2024-11-28T02:16:12,986 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:16:12,987 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:16:12,988 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:16:12,988 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:12,989 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:12,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741919_1095 (size=443) 2024-11-28T02:16:13,120 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:16:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:16:13,399 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:16:13,404 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:16:13,404 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760172004_ns3_test-17327601242692 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:13,406 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:16:13,406 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-28T02:16:13,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732760172004_ns3_test-17327601242692 table=ns3:test-17327601242692 type=FLUSH ttl=0 } in 1.3940 sec 2024-11-28T02:16:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T02:16:14,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17327601242692 completed 2024-11-28T02:16:14,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } 2024-11-28T02:16:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760174165 (current time:1732760174165). 2024-11-28T02:16:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:16:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732760174164_ns2_test-17327601242691 VERSION not specified, setting to 2 2024-11-28T02:16:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54bba3d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:14,167 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:14,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:14,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:14,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee18953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:14,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:14,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:14,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:14,168 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:14,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3468b259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:14,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:14,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:14,171 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:14,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:16:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:16:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:14,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:16:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } 2024-11-28T02:16:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-28T02:16:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:16:14,174 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:16:14,175 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:16:14,177 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:16:14,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741920_1096 (size=86) 2024-11-28T02:16:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:16:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:16:14,583 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:16:14,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d}] 2024-11-28T02:16:14,584 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:16:14,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-28T02:16:14,736 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:16:14,737 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 30e66b01c115b81cd680f3864aaf6f9d 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-28T02:16:14,741 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/.tmp/f/4fa594e15a5b48639984c047b73e32dd is 38, key is row-t20/f:q1/1732760150269/Put/seqid=0 2024-11-28T02:16:14,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741921_1097 (size=5134) 2024-11-28T02:16:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:16:15,145 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/.tmp/f/4fa594e15a5b48639984c047b73e32dd 2024-11-28T02:16:15,152 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/.tmp/f/4fa594e15a5b48639984c047b73e32dd as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/4fa594e15a5b48639984c047b73e32dd 2024-11-28T02:16:15,157 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/4fa594e15a5b48639984c047b73e32dd, entries=5, sequenceid=111, filesize=5.0 K 2024-11-28T02:16:15,158 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 30e66b01c115b81cd680f3864aaf6f9d in 422ms, sequenceid=111, compaction requested=false 2024-11-28T02:16:15,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 30e66b01c115b81cd680f3864aaf6f9d: 2024-11-28T02:16:15,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. for snapshot_1732760174164_ns2_test-17327601242691 completed. 2024-11-28T02:16:15,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d.' region-info for snapshot=snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:15,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:16:15,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/4fa594e15a5b48639984c047b73e32dd] hfiles 2024-11-28T02:16:15,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d for snapshot=snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:15,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/4fa594e15a5b48639984c047b73e32dd for snapshot=snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:15,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741922_1098 (size=130) 2024-11-28T02:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:16:15,564 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:16:15,564 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-28T02:16:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-28T02:16:15,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732760174164_ns2_test-17327601242691 on region 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:16:15,565 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:16:15,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-28T02:16:15,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 30e66b01c115b81cd680f3864aaf6f9d in 982 msec 2024-11-28T02:16:15,567 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:16:15,568 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:16:15,569 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:16:15,569 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:15,570 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:15,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741923_1099 (size=496) 2024-11-28T02:16:15,980 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:16:15,986 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:16:15,987 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760174164_ns2_test-17327601242691 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:15,988 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:16:15,988 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-28T02:16:15,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732760174164_ns2_test-17327601242691 table=ns2:test-17327601242691 type=FLUSH ttl=0 } in 1.8150 sec 2024-11-28T02:16:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T02:16:16,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17327601242691 completed 2024-11-28T02:16:16,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } 2024-11-28T02:16:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760176334 (current time:1732760176334). 2024-11-28T02:16:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:16:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732760176333_default_test-1732760124269 VERSION not specified, setting to 2 2024-11-28T02:16:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6720f6f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:16,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:16,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:16,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:16,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6607acc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:16,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:16,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:16,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:16,337 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:16,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf76bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:16,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:16,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:16,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:16,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:16:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:16:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:16,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:16:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } 2024-11-28T02:16:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-28T02:16:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:16:16,344 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:16:16,345 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:16:16,347 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:16:16,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741924_1100 (size=84) 2024-11-28T02:16:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:16:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:16:16,753 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:16:16,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure df85333d0693eea3c91e4c0b6b737054}] 2024-11-28T02:16:16,754 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:16,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for df85333d0693eea3c91e4c0b6b737054: 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. for snapshot_1732760176333_default_test-1732760124269 completed. 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054.' region-info for snapshot=snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/6f13f53cc814494699abef649aa85eca] hfiles 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 for snapshot=snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/6f13f53cc814494699abef649aa85eca for snapshot=snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/d27708e4bf90413f8d92cd5dddf8a1b4] hfiles 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 for snapshot=snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:16,906 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/d27708e4bf90413f8d92cd5dddf8a1b4 for snapshot=snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:16,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741925_1101 (size=224) 2024-11-28T02:16:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:16:17,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:17,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-28T02:16:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-28T02:16:17,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732760176333_default_test-1732760124269 on region df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:17,313 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:17,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-28T02:16:17,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure df85333d0693eea3c91e4c0b6b737054 in 561 msec 2024-11-28T02:16:17,316 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:16:17,317 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:16:17,318 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-28T02:16:17,318 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-28T02:16:17,318 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:17,319 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-28T02:16:17,319 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:17,319 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-28T02:16:17,319 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:17,320 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/mobdir/data/default/test-1732760124269/33f2a14eb25263e694b97d0d2bc810e0/mob/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054] hfiles 2024-11-28T02:16:17,320 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/mobdir/data/default/test-1732760124269/33f2a14eb25263e694b97d0d2bc810e0/mob/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:17,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741926_1102 (size=168) 2024-11-28T02:16:17,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:16:17,727 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:16:17,727 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:17,728 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:17,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741927_1103 (size=1347) 2024-11-28T02:16:18,028 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region a2d65c385b1b9e1c5b04dfa38326a47a, had cached 0 bytes from a total of 28925 2024-11-28T02:16:18,138 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:16:18,146 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:16:18,146 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760176333_default_test-1732760124269 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:18,147 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:16:18,147 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-28T02:16:18,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732760176333_default_test-1732760124269 table=test-1732760124269 type=FLUSH ttl=0 } in 1.8050 sec 2024-11-28T02:16:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T02:16:18,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732760124269 completed 2024-11-28T02:16:18,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } 2024-11-28T02:16:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760178494 (current time:1732760178494). 2024-11-28T02:16:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:16:18,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732760178494_ns4_test-17327601242693 VERSION not specified, setting to 2 2024-11-28T02:16:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a755192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:18,496 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:18,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:18,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:18,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@548c1ab2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:18,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:18,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:18,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:18,498 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:18,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7308f3c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:18,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:18,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:18,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:18,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:18,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:16:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:16:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:18,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:16:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } 2024-11-28T02:16:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-28T02:16:18,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T02:16:18,505 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:16:18,506 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:16:18,508 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:16:18,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741928_1104 (size=86) 2024-11-28T02:16:18,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T02:16:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T02:16:18,914 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:16:18,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d29139dea3e9e00e0b5beeb2ae62089f}] 2024-11-28T02:16:18,915 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:16:19,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-28T02:16:19,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:16:19,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for d29139dea3e9e00e0b5beeb2ae62089f: 2024-11-28T02:16:19,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. for snapshot_1732760178494_ns4_test-17327601242693 completed. 2024-11-28T02:16:19,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f.' region-info for snapshot=snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:19,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:16:19,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-28T02:16:19,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741929_1105 (size=52) 2024-11-28T02:16:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T02:16:19,429 INFO [master/d9827d9b919f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T02:16:19,429 INFO [master/d9827d9b919f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T02:16:19,473 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:16:19,473 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-28T02:16:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-28T02:16:19,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732760178494_ns4_test-17327601242693 on region d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:16:19,474 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:16:19,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-28T02:16:19,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d29139dea3e9e00e0b5beeb2ae62089f in 561 msec 2024-11-28T02:16:19,476 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:16:19,477 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:16:19,478 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:16:19,478 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:19,478 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:19,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741930_1106 (size=443) 2024-11-28T02:16:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T02:16:19,888 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:16:19,893 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:16:19,893 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_1732760178494_ns4_test-17327601242693 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:19,895 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:16:19,895 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-28T02:16:19,896 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732760178494_ns4_test-17327601242693 table=ns4:test-17327601242693 type=FLUSH ttl=0 } in 1.3920 sec 2024-11-28T02:16:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T02:16:20,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17327601242693 completed 2024-11-28T02:16:20,654 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732760165539 2024-11-28T02:16:20,654 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-28T02:16:20,654 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-28T02:16:20,654 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732760165539_backup:system_bulk 2024-11-28T02:16:20,654 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732760169846_backup_system_bulk to hdfs://localhost:38631/backupUT/backup_1732760165539/backup/system_bulk/ 2024-11-28T02:16:20,654 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-28T02:16:20,686 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:38631, inputRoot=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:16:20,686 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_128661675_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38631/backupUT/backup_1732760165539/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38631/backupUT/backup_1732760165539/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:20,688 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-28T02:16:20,691 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760169846_backup_system_bulk to hdfs://localhost:38631/backupUT/backup_1732760165539/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:20,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741931_1107 (size=76) 2024-11-28T02:16:20,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741932_1108 (size=702) 2024-11-28T02:16:21,105 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:21,106 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:21,106 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-17423141232306745399.jar 2024-11-28T02:16:22,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:22,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:16:22,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:16:22,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:16:22,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:16:22,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:16:22,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:16:22,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:16:22,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:16:22,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:16:22,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:16:22,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:16:22,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:22,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:22,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:22,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:22,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:22,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:22,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:22,141 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:16:22,169 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732760169846_backup_system_bulk' hfile list 2024-11-28T02:16:22,260 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local394563034_0005_2828af13-9b23-4743-8ef8-51177e51b882/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-28T02:16:22,260 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local394563034_0005_2828af13-9b23-4743-8ef8-51177e51b882/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-28T02:16:23,300 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-28T02:16:23,301 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-28T02:16:23,306 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:23,306 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732760169846_backup_system_bulk finished. 2024-11-28T02:16:23,306 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732760165539_ns3:test-17327601242692 2024-11-28T02:16:23,306 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732760172004_ns3_test-17327601242692 to hdfs://localhost:38631/backupUT/backup_1732760165539/ns3/test-17327601242692/ 2024-11-28T02:16:23,306 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-28T02:16:23,337 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:38631, inputRoot=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:16:23,337 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_128661675_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38631/backupUT/backup_1732760165539/ns3/test-17327601242692, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38631/backupUT/backup_1732760165539/ns3/test-17327601242692/.hbase-snapshot/.tmp/snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:23,339 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-28T02:16:23,343 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760172004_ns3_test-17327601242692 to hdfs://localhost:38631/backupUT/backup_1732760165539/ns3/test-17327601242692/.hbase-snapshot/.tmp/snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:23,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741933_1109 (size=86) 2024-11-28T02:16:23,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741934_1110 (size=443) 2024-11-28T02:16:23,755 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:23,756 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:23,756 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,737 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-6039222712691343560.jar 2024-11-28T02:16:24,737 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,737 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,738 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,738 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,738 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,738 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,739 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,739 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,739 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:24,739 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:16:24,739 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:16:24,740 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:16:24,740 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:16:24,740 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:16:24,740 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:16:24,740 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:16:24,741 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:16:24,741 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:16:24,741 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:16:24,741 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:16:24,742 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:24,742 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:24,742 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:24,742 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:24,742 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:24,743 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:24,743 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:24,783 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:16:24,809 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732760172004_ns3_test-17327601242692' hfile list 2024-11-28T02:16:24,896 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1539741780_0006_52ae7996-aaba-4399-87d7-af79c3bdb7b4/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-28T02:16:24,896 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1539741780_0006_52ae7996-aaba-4399-87d7-af79c3bdb7b4/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-28T02:16:25,936 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-28T02:16:25,938 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-28T02:16:25,942 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:25,942 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732760172004_ns3_test-17327601242692 finished. 2024-11-28T02:16:25,942 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732760165539_ns2:test-17327601242691 2024-11-28T02:16:25,942 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732760174164_ns2_test-17327601242691 to hdfs://localhost:38631/backupUT/backup_1732760165539/ns2/test-17327601242691/ 2024-11-28T02:16:25,943 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-28T02:16:25,973 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:38631, inputRoot=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:16:25,973 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_128661675_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38631/backupUT/backup_1732760165539/ns2/test-17327601242691, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38631/backupUT/backup_1732760165539/ns2/test-17327601242691/.hbase-snapshot/.tmp/snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:25,975 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-28T02:16:25,980 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760174164_ns2_test-17327601242691 to hdfs://localhost:38631/backupUT/backup_1732760165539/ns2/test-17327601242691/.hbase-snapshot/.tmp/snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:25,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741935_1111 (size=86) 2024-11-28T02:16:25,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741936_1112 (size=496) 2024-11-28T02:16:26,391 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:26,392 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:26,392 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,375 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-10398190529438066785.jar 2024-11-28T02:16:27,376 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,376 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,376 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,376 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,377 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,377 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,377 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,377 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,377 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:27,378 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:16:27,378 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:16:27,378 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:16:27,378 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:16:27,378 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:16:27,379 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:16:27,379 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:16:27,379 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:16:27,380 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:16:27,380 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:16:27,380 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:16:27,380 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:27,381 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:27,381 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:27,381 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:27,381 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:27,381 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:27,382 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:27,423 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:16:27,450 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732760174164_ns2_test-17327601242691' hfile list 2024-11-28T02:16:27,452 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-11-28T02:16:27,536 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1518952774_0007_7c39cfa3-35e4-4320-939c-49785b0e27d0/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-28T02:16:27,536 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1518952774_0007_7c39cfa3-35e4-4320-939c-49785b0e27d0/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-28T02:16:27,616 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-28T02:16:27,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741937_1113 (size=8413) 2024-11-28T02:16:28,062 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d output=hdfs://localhost:38631/backupUT/backup_1732760165539/ns2/test-17327601242691/archive/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/1a094c37607b4bc7a971b480377e582d 2024-11-28T02:16:28,062 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-28T02:16:28,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741938_1114 (size=5134) 2024-11-28T02:16:28,494 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/4fa594e15a5b48639984c047b73e32dd output=hdfs://localhost:38631/backupUT/backup_1732760165539/ns2/test-17327601242691/archive/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/f/4fa594e15a5b48639984c047b73e32dd 2024-11-28T02:16:28,494 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-11-28T02:16:28,576 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-28T02:16:28,577 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-28T02:16:28,584 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:28,584 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732760174164_ns2_test-17327601242691 finished. 2024-11-28T02:16:28,585 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732760165539_test-1732760124269 2024-11-28T02:16:28,585 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732760176333_default_test-1732760124269 to hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/ 2024-11-28T02:16:28,585 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-28T02:16:28,619 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:38631, inputRoot=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:16:28,619 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_128661675_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/.hbase-snapshot/.tmp/snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:28,621 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-28T02:16:28,627 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760176333_default_test-1732760124269 to hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/.hbase-snapshot/.tmp/snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:28,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741939_1115 (size=84) 2024-11-28T02:16:28,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741940_1116 (size=1347) 2024-11-28T02:16:29,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:29,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:29,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,023 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-7768818101955648458.jar 2024-11-28T02:16:30,023 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,023 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,024 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:30,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:16:30,025 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:16:30,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:16:30,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:16:30,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:16:30,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:16:30,026 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:16:30,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:16:30,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:16:30,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:16:30,027 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:16:30,028 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:30,028 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:30,028 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:30,028 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:30,028 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:30,029 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:30,029 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:30,069 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:16:30,095 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732760176333_default_test-1732760124269' hfile list 2024-11-28T02:16:30,097 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-11-28T02:16:30,184 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1415692023_0008_c88af0b1-9205-4912-b5c4-867f17c167ce/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-28T02:16:30,184 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local1415692023_0008_c88af0b1-9205-4912-b5c4-867f17c167ce/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-28T02:16:30,253 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-28T02:16:30,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741941_1117 (size=13760) 2024-11-28T02:16:30,694 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/d27708e4bf90413f8d92cd5dddf8a1b4 output=hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/archive/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/d27708e4bf90413f8d92cd5dddf8a1b4 2024-11-28T02:16:30,695 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-28T02:16:30,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741942_1118 (size=10796) 2024-11-28T02:16:31,127 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/6f13f53cc814494699abef649aa85eca output=hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/archive/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/6f13f53cc814494699abef649aa85eca 2024-11-28T02:16:31,127 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-28T02:16:31,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741943_1119 (size=8413) 2024-11-28T02:16:31,567 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 output=hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/archive/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f/001e01fc909c46b5bb46d75f30451132 2024-11-28T02:16:31,567 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-28T02:16:31,592 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/mobdir/data/default/test-1732760124269/33f2a14eb25263e694b97d0d2bc810e0/mob/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054 to hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/archive/data/default/test-1732760124269/33f2a14eb25263e694b97d0d2bc810e0/mob/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054" truncated to max limit (512 characters) 2024-11-28T02:16:31,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741944_1120 (size=5591) 2024-11-28T02:16:32,000 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/mobdir/data/default/test-1732760124269/33f2a14eb25263e694b97d0d2bc810e0/mob/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054 output=hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/archive/data/default/test-1732760124269/33f2a14eb25263e694b97d0d2bc810e0/mob/d41d8cd98f00b204e9800998ecf8427e20241128bec28c5c58a64eecaa8e4afd2b08d779_df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:32,000 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.012M/sec 2024-11-28T02:16:32,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741945_1121 (size=5484) 2024-11-28T02:16:32,440 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 output=hdfs://localhost:38631/backupUT/backup_1732760165539/default/test-1732760124269/archive/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/mob/e81c569e82b946cdb07c47cdb39a2d86 2024-11-28T02:16:32,440 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-28T02:16:33,226 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-28T02:16:33,227 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-28T02:16:33,234 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:33,234 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732760176333_default_test-1732760124269 finished. 2024-11-28T02:16:33,234 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732760165539_ns4:test-17327601242693 2024-11-28T02:16:33,235 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732760178494_ns4_test-17327601242693 to hdfs://localhost:38631/backupUT/backup_1732760165539/ns4/test-17327601242693/ 2024-11-28T02:16:33,235 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-28T02:16:33,264 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:38631, inputRoot=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96 2024-11-28T02:16:33,264 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_128661675_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38631/backupUT/backup_1732760165539/ns4/test-17327601242693, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38631/backupUT/backup_1732760165539/ns4/test-17327601242693/.hbase-snapshot/.tmp/snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:33,265 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-28T02:16:33,269 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_1732760178494_ns4_test-17327601242693 to hdfs://localhost:38631/backupUT/backup_1732760165539/ns4/test-17327601242693/.hbase-snapshot/.tmp/snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:33,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741946_1122 (size=86) 2024-11-28T02:16:33,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741947_1123 (size=443) 2024-11-28T02:16:33,680 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:33,680 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:33,680 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,661 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-7668254335503229230.jar 2024-11-28T02:16:34,661 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,661 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,662 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,662 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,662 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:34,663 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:16:34,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:16:34,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:16:34,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:16:34,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:16:34,664 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:16:34,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:16:34,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:16:34,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:16:34,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:16:34,665 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:16:34,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:34,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:34,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:34,666 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:34,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-28T02:16:34,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:34,667 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:34,707 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:16:34,734 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732760178494_ns4_test-17327601242693' hfile list 2024-11-28T02:16:34,834 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local744488908_0009_1dc2bdf0-e6b5-4a03-b478-adf63d5e12e1/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-28T02:16:34,834 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/mapreduce.cluster.local.dir/job_local744488908_0009_1dc2bdf0-e6b5-4a03-b478-adf63d5e12e1/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-28T02:16:35,875 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-28T02:16:35,876 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-28T02:16:35,881 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:35,881 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732760178494_ns4_test-17327601242693 finished. 2024-11-28T02:16:35,881 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-28T02:16:35,881 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17327601242692 2024-11-28T02:16:35,881 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17327601242691 2024-11-28T02:16:35,881 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732760124269 2024-11-28T02:16:35,881 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17327601242693 2024-11-28T02:16:35,890 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732760165539 2024-11-28T02:16:35,890 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-28T02:16:35,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741948_1124 (size=209) 2024-11-28T02:16:36,296 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38631/backupUT/backup_1732760165539/.backup.manifest 2024-11-28T02:16:36,296 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732760165539 finished: type=FULL,tablelist=backup:system_bulk;ns3:test-17327601242692;ns2:test-17327601242691;test-1732760124269;ns4:test-17327601242693,targetRootDir=hdfs://localhost:38631/backupUT,startts=1732760169724,completets=1732760195890,bytescopied=0 2024-11-28T02:16:36,296 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-28T02:16:36,296 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:36,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732760169846_backup_system_bulk" type: DISABLED 2024-11-28T02:16:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732760169846_backup_system_bulk 2024-11-28T02:16:36,300 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732760169846_backup_system_bulk for backup backup_1732760165539 succeeded. 2024-11-28T02:16:36,300 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:36,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732760172004_ns3_test-17327601242692" type: DISABLED 2024-11-28T02:16:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732760172004_ns3_test-17327601242692 2024-11-28T02:16:36,303 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732760172004_ns3_test-17327601242692 for backup backup_1732760165539 succeeded. 2024-11-28T02:16:36,303 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:36,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732760174164_ns2_test-17327601242691" type: DISABLED 2024-11-28T02:16:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732760174164_ns2_test-17327601242691 2024-11-28T02:16:36,306 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732760174164_ns2_test-17327601242691 for backup backup_1732760165539 succeeded. 2024-11-28T02:16:36,306 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:36,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732760176333_default_test-1732760124269" type: DISABLED 2024-11-28T02:16:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732760176333_default_test-1732760124269 2024-11-28T02:16:36,308 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732760176333_default_test-1732760124269 for backup backup_1732760165539 succeeded. 2024-11-28T02:16:36,308 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:36,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732760178494_ns4_test-17327601242693" type: DISABLED 2024-11-28T02:16:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732760178494_ns4_test-17327601242693 2024-11-28T02:16:36,311 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732760178494_ns4_test-17327601242693 for backup backup_1732760165539 succeeded. 2024-11-28T02:16:36,312 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-28T02:16:36,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-28T02:16:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-28T02:16:36,318 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-28T02:16:36,320 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-28T02:16:36,325 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732760165539 completed. 2024-11-28T02:16:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:36,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,385 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-28T02:16:41,389 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732760201348 2024-11-28T02:16:41,391 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732760165539} 2024-11-28T02:16:41,391 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-28T02:16:41,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:41,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:16:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760201482 (current time:1732760201482). 2024-11-28T02:16:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:16:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-28T02:16:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129bf997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:41,484 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:41,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:41,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:41,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85feed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:41,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:41,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:41,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:41,486 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:41,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586402fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:41,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:41,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:41,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:41,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:16:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:16:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:41,490 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:16:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:16:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-28T02:16:41,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:16:41,492 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:16:41,493 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:16:41,495 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:16:41,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741949_1125 (size=52) 2024-11-28T02:16:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:16:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:16:41,901 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-28T02:16:41,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a}] 2024-11-28T02:16:41,902 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:42,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42127 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-28T02:16:42,053 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:42,054 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing a2d65c385b1b9e1c5b04dfa38326a47a 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-28T02:16:42,059 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/285993cd51774e86a2bc07a5f816cc20 is 158, key is trslm:hdfs://localhost:38631/backupUT\x00ns2:test-17327601242691/meta:log-roll-map/1732760195885/Put/seqid=0 2024-11-28T02:16:42,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741950_1126 (size=6646) 2024-11-28T02:16:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:16:42,463 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/285993cd51774e86a2bc07a5f816cc20 2024-11-28T02:16:42,472 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 is 826, key is session:backup_1732760165539/session:context/1732760196319/Put/seqid=0 2024-11-28T02:16:42,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741951_1127 (size=5928) 2024-11-28T02:16:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:16:42,877 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 2024-11-28T02:16:42,883 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/285993cd51774e86a2bc07a5f816cc20 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20 2024-11-28T02:16:42,888 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20, entries=12, sequenceid=42, filesize=6.5 K 2024-11-28T02:16:42,889 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 2024-11-28T02:16:42,894 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7, entries=2, sequenceid=42, filesize=5.8 K 2024-11-28T02:16:42,895 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for a2d65c385b1b9e1c5b04dfa38326a47a in 840ms, sequenceid=42, compaction requested=true 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for a2d65c385b1b9e1c5b04dfa38326a47a: 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. for snapshot_backup_system completed. 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.' region-info for snapshot=snapshot_backup_system 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20] hfiles 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17 for snapshot=snapshot_backup_system 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935 for snapshot=snapshot_backup_system 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20 for snapshot=snapshot_backup_system 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7] hfiles 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba for snapshot=snapshot_backup_system 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a for snapshot=snapshot_backup_system 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486 for snapshot=snapshot_backup_system 2024-11-28T02:16:42,895 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 for snapshot=snapshot_backup_system 2024-11-28T02:16:42,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741952_1128 (size=330) 2024-11-28T02:16:43,120 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T02:16:43,301 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:43,301 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-28T02:16:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-28T02:16:43,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:43,302 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:43,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-28T02:16:43,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a in 1.4020 sec 2024-11-28T02:16:43,304 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-28T02:16:43,305 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:16:43,306 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:16:43,306 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-28T02:16:43,307 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-28T02:16:43,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741953_1129 (size=967) 2024-11-28T02:16:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:16:43,717 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:16:43,724 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:16:43,725 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/snapshot_backup_system 2024-11-28T02:16:43,726 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:16:43,726 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-28T02:16:43,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2360 sec 2024-11-28T02:16:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T02:16:45,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-28T02:16:45,653 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732760201348 started at 1732760205653. 2024-11-28T02:16:45,655 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732760201348 has been started. 2024-11-28T02:16:45,660 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, ns2:test-17327601242691, ns3:test-17327601242692, ns4:test-17327601242693, test-1732760124269] 2024-11-28T02:16:45,663 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732760155958for backupID backup_1732760201348 2024-11-28T02:16:45,663 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-28T02:16:45,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-28T02:16:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-28T02:16:45,664 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-28T02:16:45,664 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-28T02:16:45,664 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-28T02:16:45,664 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-28T02:16:45,664 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,664 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:45,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-28T02:16:45,670 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,670 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-28T02:16:45,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:16:45,670 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,670 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-28T02:16:45,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:45,671 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-28T02:16:45,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:45,671 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-28T02:16:45,671 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-28T02:16:45,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-28T02:16:45,671 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-28T02:16:45,671 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-28T02:16:45,671 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-28T02:16:45,672 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-28T02:16:45,672 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'd9827d9b919f,42127,1732760115598' joining acquired barrier for procedure (rolllog) in zk 2024-11-28T02:16:45,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,673 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:45,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:16:45,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:16:45,674 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-28T02:16:45,674 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-28T02:16:45,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:16:45,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:16:45,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:45,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:16:45,675 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'd9827d9b919f,42127,1732760115598' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-28T02:16:45,675 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1dfe2088[Count = 0] remaining members to acquire global barrier 2024-11-28T02:16:45,675 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-28T02:16:45,675 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:45,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:45,677 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:45,677 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:45,677 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-28T02:16:45,677 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,677 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-28T02:16:45,677 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-28T02:16:45,677 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,677 INFO [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732760169779 highest: 1732760169779 on d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,677 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.meta:.meta(num 1732760169745) roll requested 2024-11-28T02:16:45,691 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760205678.meta, exclude list is [], retry=0 2024-11-28T02:16:45,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:16:45,695 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760169745.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760205678.meta 2024-11-28T02:16:45,695 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:16:45,695 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760169745.meta is not closed yet, will try archiving it next time 2024-11-28T02:16:45,695 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1:(num 1732760169762) roll requested 2024-11-28T02:16:45,696 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760169745.meta not finished, retry = 0 2024-11-28T02:16:45,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741911_1087 (size=93) 2024-11-28T02:16:45,708 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760205695, exclude list is [], retry=0 2024-11-28T02:16:45,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:16:45,711 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760169762 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760205695 2024-11-28T02:16:45,711 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:16:45,711 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760169762 is not closed yet, will try archiving it next time 2024-11-28T02:16:45,711 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0:(num 1732760169779) roll requested 2024-11-28T02:16:45,711 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760155938 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760155938 2024-11-28T02:16:45,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741912_1088 (size=4710) 2024-11-28T02:16:45,713 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760169762 not finished, retry = 0 2024-11-28T02:16:45,724 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760205711, exclude list is [], retry=0 2024-11-28T02:16:45,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37989,DS-fd485f3a-3fa4-4e7a-b571-e7bbb00c2bb8,DISK] 2024-11-28T02:16:45,727 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760205711 2024-11-28T02:16:45,728 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41009:41009)] 2024-11-28T02:16:45,728 DEBUG [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 is not closed yet, will try archiving it next time 2024-11-28T02:16:45,728 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760136284 2024-11-28T02:16:45,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741913_1089 (size=653) 2024-11-28T02:16:45,729 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 not finished, retry = 0 2024-11-28T02:16:45,738 DEBUG [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-28T02:16:45,738 INFO [rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732760205711 on d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-28T02:16:45,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:45,754 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-28T02:16:45,754 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'd9827d9b919f,42127,1732760115598' in zk 2024-11-28T02:16:45,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,757 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-28T02:16:45,757 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-28T02:16:45,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:16:45,757 DEBUG [member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-28T02:16:45,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:16:45,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:16:45,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:16:45,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:45,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:16:45,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:45,760 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,760 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'd9827d9b919f,42127,1732760115598': 2024-11-28T02:16:45,760 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'd9827d9b919f,42127,1732760115598' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-28T02:16:45,760 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-28T02:16:45,760 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-28T02:16:45,761 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-28T02:16:45,761 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-28T02:16:45,761 INFO [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-28T02:16:45,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-28T02:16:45,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,763 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-28T02:16:45,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-28T02:16:45,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-28T02:16:45,763 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-28T02:16:45,763 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:16:45,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:45,764 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-28T02:16:45,764 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:45,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-28T02:16:45,765 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,765 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-28T02:16:45,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:45,768 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-28T02:16:45,768 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-28T02:16:45,768 DEBUG [(d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-28T02:16:45,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-28T02:16:45,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-28T02:16:45,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-28T02:16:45,769 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-28T02:16:45,769 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-28T02:16:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-28T02:16:45,775 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {d9827d9b919f:42127=1732760155958} newestTimestamps: {d9827d9b919f:42127=1732760169779} 2024-11-28T02:16:45,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: d9827d9b919f%2C42127%2C1732760115598.meta.1732760119169.meta 2024-11-28T02:16:45,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: d9827d9b919f%2C42127%2C1732760115598.meta.1732760155910.meta 2024-11-28T02:16:45,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: d9827d9b919f%2C42127%2C1732760115598.meta.1732760169745.meta 2024-11-28T02:16:45,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: d9827d9b919f%2C42127%2C1732760115598.meta.1732760205678.meta 2024-11-28T02:16:45,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 2024-11-28T02:16:45,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760205711 2024-11-28T02:16:45,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760169762 2024-11-28T02:16:45,777 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760205695 2024-11-28T02:16:45,778 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.meta.1732760136242.meta 2024-11-28T02:16:45,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741957_1133 (size=640) 2024-11-28T02:16:45,797 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760169745.meta to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.meta.1732760169745.meta 2024-11-28T02:16:45,830 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 2024-11-28T02:16:46,188 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:38631/backupUT/backup_1732760201348/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:16:46,188 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-28T02:16:46,190 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-28T02:16:46,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741958_1134 (size=45) 2024-11-28T02:16:46,595 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-28T02:16:46,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741959_1135 (size=390) 2024-11-28T02:16:47,006 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17327601242692 target: hdfs://localhost:38631/backupUT/backup_1732760201348/ns3/test-17327601242692 descriptor: 'ns3:test-17327601242692', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:16:47,006 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-28T02:16:47,008 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17327601242692 2024-11-28T02:16:47,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741960_1136 (size=50) 2024-11-28T02:16:47,413 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17327601242692 2024-11-28T02:16:47,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741961_1137 (size=364) 2024-11-28T02:16:47,823 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17327601242691 target: hdfs://localhost:38631/backupUT/backup_1732760201348/ns2/test-17327601242691 descriptor: 'ns2:test-17327601242691', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:16:47,823 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-28T02:16:47,825 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17327601242691 2024-11-28T02:16:47,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741962_1138 (size=50) 2024-11-28T02:16:48,231 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17327601242691 2024-11-28T02:16:48,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741963_1139 (size=950) 2024-11-28T02:16:48,641 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732760124269 target: hdfs://localhost:38631/backupUT/backup_1732760201348/default/test-1732760124269 descriptor: 'test-1732760124269', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:16:48,641 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-28T02:16:48,643 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732760124269 2024-11-28T02:16:48,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741964_1140 (size=53) 2024-11-28T02:16:49,048 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732760124269 2024-11-28T02:16:49,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741965_1141 (size=390) 2024-11-28T02:16:49,458 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17327601242693 target: hdfs://localhost:38631/backupUT/backup_1732760201348/ns4/test-17327601242693 descriptor: 'ns4:test-17327601242693', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T02:16:49,458 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-28T02:16:49,461 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17327601242693 2024-11-28T02:16:49,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741966_1142 (size=50) 2024-11-28T02:16:49,866 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17327601242693 2024-11-28T02:16:49,876 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:38631/backupUT/.tmp/backup_1732760201348 from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779;hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1.1732760169762 2024-11-28T02:16:49,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c2a4966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:49,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:49,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:49,878 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:49,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:49,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:49,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586e7595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:49,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:49,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:49,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:49,879 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:49,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4ab3c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:49,880 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T02:16:49,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is d9827d9b919f,43413,1732760114080 2024-11-28T02:16:49,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b9327be 2024-11-28T02:16:49,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T02:16:49,881 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T02:16:49,884 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-28T02:16:49,884 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-28T02:16:49,885 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-28T02:16:49,885 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:49,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:49,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:49,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:49,888 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-28T02:16:49,888 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17327601242691 2024-11-28T02:16:49,888 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17327601242691,, stopping at row=ns2:test-17327601242691 ,, for max=2147483647 with caching=100 2024-11-28T02:16:49,890 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17327601242691: ns2:test-17327601242691; 2024-11-28T02:16:49,890 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17327601242692 2024-11-28T02:16:49,890 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17327601242692,, stopping at row=ns3:test-17327601242692 ,, for max=2147483647 with caching=100 2024-11-28T02:16:49,891 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17327601242692: ns3:test-17327601242692; 2024-11-28T02:16:49,891 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17327601242693 2024-11-28T02:16:49,891 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17327601242693,, stopping at row=ns4:test-17327601242693 ,, for max=2147483647 with caching=100 2024-11-28T02:16:49,893 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17327601242693: ns4:test-17327601242693; 2024-11-28T02:16:49,893 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732760124269 2024-11-28T02:16:49,893 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732760124269,, stopping at row=test-1732760124269 ,, for max=2147483647 with caching=100 2024-11-28T02:16:49,895 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732760124269: test-1732760124269; 2024-11-28T02:16:49,895 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-28T02:16:49,895 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/hbase-staging/partitions_1a652f32-fad3-4f72-bc54-ae1158cf42c2 2024-11-28T02:16:49,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741967_1143 (size=324) 2024-11-28T02:16:50,301 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:50,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:50,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:50,384 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region df85333d0693eea3c91e4c0b6b737054, had cached 0 bytes from a total of 38453 2024-11-28T02:16:51,281 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-2499273230774763477.jar 2024-11-28T02:16:51,281 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:16:51,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-28T02:16:51,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-28T02:16:51,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-28T02:16:51,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-28T02:16:51,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-28T02:16:51,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-28T02:16:51,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-28T02:16:51,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-28T02:16:51,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-28T02:16:51,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-28T02:16:51,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-28T02:16:51,287 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-28T02:16:51,287 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,ns2:test-17327601242691,ns3:test-17327601242692,ns4:test-17327601242693,default:test-1732760124269 2024-11-28T02:16:51,287 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:16:51,287 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:51,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:51,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:51,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:51,287 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-28T02:16:52,267 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop-11285871983413620624.jar 2024-11-28T02:16:52,279 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-28T02:16:52,303 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 for WAL files 2024-11-28T02:16:52,304 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1732760201348File hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 does not exist. java.io.FileNotFoundException: File hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T02:16:52,305 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732760201348,startts=1732760205653,failedts=1732760212305,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760169779 does not exist. 2024-11-28T02:16:52,305 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-28T02:16:52,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable backup:system 2024-11-28T02:16:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-11-28T02:16:52,321 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760212321"}]},"ts":"1732760212321"} 2024-11-28T02:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T02:16:52,323 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-11-28T02:16:52,323 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-11-28T02:16:52,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-11-28T02:16:52,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, UNASSIGN}] 2024-11-28T02:16:52,329 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, UNASSIGN 2024-11-28T02:16:52,330 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=a2d65c385b1b9e1c5b04dfa38326a47a, regionState=CLOSING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:16:52,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, UNASSIGN because future has completed 2024-11-28T02:16:52,332 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T02:16:52,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:16:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T02:16:52,484 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:52,484 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T02:16:52,485 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing a2d65c385b1b9e1c5b04dfa38326a47a, disabling compactions & flushes 2024-11-28T02:16:52,485 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:52,485 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:52,485 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. after waiting 0 ms 2024-11-28T02:16:52,485 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:52,485 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing a2d65c385b1b9e1c5b04dfa38326a47a 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-11-28T02:16:52,489 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/2bcd6239c2834087b070effe5dce8997 is 103, key is rslogts:hdfs://localhost:38631/backupUT\x00d9827d9b919f:42127/meta:rs-log-ts/1732760205753/Put/seqid=0 2024-11-28T02:16:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741968_1144 (size=5338) 2024-11-28T02:16:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T02:16:52,893 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/2bcd6239c2834087b070effe5dce8997 2024-11-28T02:16:52,901 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/b9facb36dcbd4caba025b31c0e332717 is 303, key is session:backup_1732760201348/session:context/1732760205654/Put/seqid=0 2024-11-28T02:16:52,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741969_1145 (size=5391) 2024-11-28T02:16:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T02:16:53,306 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/b9facb36dcbd4caba025b31c0e332717 2024-11-28T02:16:53,311 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/2bcd6239c2834087b070effe5dce8997 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/2bcd6239c2834087b070effe5dce8997 2024-11-28T02:16:53,316 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/2bcd6239c2834087b070effe5dce8997, entries=1, sequenceid=47, filesize=5.2 K 2024-11-28T02:16:53,317 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/b9facb36dcbd4caba025b31c0e332717 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/b9facb36dcbd4caba025b31c0e332717 2024-11-28T02:16:53,322 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/b9facb36dcbd4caba025b31c0e332717, entries=1, sequenceid=47, filesize=5.3 K 2024-11-28T02:16:53,323 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for a2d65c385b1b9e1c5b04dfa38326a47a in 838ms, sequenceid=47, compaction requested=true 2024-11-28T02:16:53,327 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-11-28T02:16:53,327 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:53,328 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:53,328 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for a2d65c385b1b9e1c5b04dfa38326a47a: Waiting for close lock at 1732760212484Running coprocessor pre-close hooks at 1732760212484Disabling compacts and flushes for region at 1732760212484Disabling writes for close at 1732760212485 (+1 ms)Obtaining lock to block concurrent updates at 1732760212485Preparing flush snapshotting stores in a2d65c385b1b9e1c5b04dfa38326a47a at 1732760212485Finished memstore snapshotting backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1732760212485Flushing stores of backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. at 1732760212486 (+1 ms)Flushing a2d65c385b1b9e1c5b04dfa38326a47a/meta: creating writer at 1732760212486Flushing a2d65c385b1b9e1c5b04dfa38326a47a/meta: appending metadata at 1732760212488 (+2 ms)Flushing a2d65c385b1b9e1c5b04dfa38326a47a/meta: closing flushed file at 1732760212488Flushing a2d65c385b1b9e1c5b04dfa38326a47a/session: creating writer at 1732760212899 (+411 ms)Flushing a2d65c385b1b9e1c5b04dfa38326a47a/session: appending metadata at 1732760212901 (+2 ms)Flushing a2d65c385b1b9e1c5b04dfa38326a47a/session: closing flushed file at 1732760212901Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b2f06dc: reopening flushed file at 1732760213311 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b55bd24: reopening flushed file at 1732760213316 (+5 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for a2d65c385b1b9e1c5b04dfa38326a47a in 838ms, sequenceid=47, compaction requested=true at 1732760213323 (+7 ms)Writing region close event to WAL at 1732760213324 (+1 ms)Running coprocessor post-close hooks at 1732760213327 (+3 ms)Closed at 1732760213328 (+1 ms) 2024-11-28T02:16:53,330 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:53,331 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=a2d65c385b1b9e1c5b04dfa38326a47a, regionState=CLOSED 2024-11-28T02:16:53,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:16:53,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-28T02:16:53,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598 in 1.0010 sec 2024-11-28T02:16:53,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-28T02:16:53,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, UNASSIGN in 1.0060 sec 2024-11-28T02:16:53,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-28T02:16:53,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0130 sec 2024-11-28T02:16:53,340 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760213340"}]},"ts":"1732760213340"} 2024-11-28T02:16:53,342 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-11-28T02:16:53,342 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-11-28T02:16:53,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0280 sec 2024-11-28T02:16:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-28T02:16:53,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-11-28T02:16:53,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732760213471 2024-11-28T02:16:53,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:16:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732760213471 (current time:1732760213471). 2024-11-28T02:16:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-28T02:16:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1732760213471 VERSION not specified, setting to 2 2024-11-28T02:16:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e550ef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ClusterIdFetcher(90): Going to request d9827d9b919f,43413,-1 for getting cluster id 2024-11-28T02:16:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T02:16:53,473 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e93b2abd-558d-40dd-8d9f-9c3f791c7807' 2024-11-28T02:16:53,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T02:16:53,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e93b2abd-558d-40dd-8d9f-9c3f791c7807" 2024-11-28T02:16:53,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441fcc27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:53,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9827d9b919f,43413,-1] 2024-11-28T02:16:53,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T02:16:53,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:53,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T02:16:53,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458c258c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-28T02:16:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T02:16:53,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9827d9b919f,42127,1732760115598, seqNum=-1] 2024-11-28T02:16:53,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T02:16:53,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T02:16:53,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413. 2024-11-28T02:16:53,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T02:16:53,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:53,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:53,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-28T02:16:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:16:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-28T02:16:53,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:16:53,479 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-28T02:16:53,480 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-28T02:16:53,482 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-28T02:16:53,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741970_1146 (size=81) 2024-11-28T02:16:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:16:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:16:53,888 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-11-28T02:16:53,889 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-28T02:16:53,889 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-28T02:16:53,889 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:53,891 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/2bcd6239c2834087b070effe5dce8997, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17] hfiles 2024-11-28T02:16:53,891 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20 2024-11-28T02:16:53,891 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/2bcd6239c2834087b070effe5dce8997 2024-11-28T02:16:53,891 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935 2024-11-28T02:16:53,891 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17 2024-11-28T02:16:53,891 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:53,893 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/b9facb36dcbd4caba025b31c0e332717, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486] hfiles 2024-11-28T02:16:53,893 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba 2024-11-28T02:16:53,893 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a 2024-11-28T02:16:53,893 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/b9facb36dcbd4caba025b31c0e332717 2024-11-28T02:16:53,893 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 2024-11-28T02:16:53,893 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486 2024-11-28T02:16:53,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741971_1147 (size=409) 2024-11-28T02:16:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:16:54,298 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => a2d65c385b1b9e1c5b04dfa38326a47a, NAME => 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-11-28T02:16:54,299 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-28T02:16:54,300 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-28T02:16:54,300 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1732760213471 2024-11-28T02:16:54,301 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732760213471 2024-11-28T02:16:54,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741972_1148 (size=1046) 2024-11-28T02:16:54,509 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-28T02:16:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:16:54,711 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-28T02:16:54,720 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-28T02:16:54,720 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732760213471 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1732760213471 2024-11-28T02:16:54,721 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-28T02:16:54,721 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-28T02:16:54,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732760213471 table=backup:system type=FLUSH ttl=0 } in 1.2430 sec 2024-11-28T02:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-28T02:16:55,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-28T02:16:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732760201482 type: FLUSH version: 2 ttl: 0 ) 2024-11-28T02:16:55,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-11-28T02:16:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-28T02:16:55,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741973_1149 (size=635) 2024-11-28T02:16:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-28T02:16:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-28T02:16:56,057 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-11-28T02:16:56,057 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-11-28T02:16:56,058 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-28T02:16:56,063 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732760201482 type: FLUSH version: 2 ttl: 0 2024-11-28T02:16:56,064 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system 2024-11-28T02:16:56,066 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-11-28T02:16:56,066 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(229): region to restore: a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,070 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:56,074 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/2bcd6239c2834087b070effe5dce8997 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/2bcd6239c2834087b070effe5dce8997 2024-11-28T02:16:56,074 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:56,077 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/b9facb36dcbd4caba025b31c0e332717 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/b9facb36dcbd4caba025b31c0e332717 2024-11-28T02:16:56,077 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732760201482 type: FLUSH version: 2 ttl: 0 2024-11-28T02:16:56,078 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732760216078"}]},"ts":"9223372036854775807"} 2024-11-28T02:16:56,082 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T02:16:56,082 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a2d65c385b1b9e1c5b04dfa38326a47a, NAME => 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T02:16:56,082 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732760216079"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732760216079"}]},"ts":"1732760216079"} 2024-11-28T02:16:56,083 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T02:16:56,084 INFO [PEWorker-1 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-11-28T02:16:56,084 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => a2d65c385b1b9e1c5b04dfa38326a47a, NAME => 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T02:16:56,084 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-11-28T02:16:56,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732760201482 type: FLUSH version: 2 ttl: 0 ) in 449 msec 2024-11-28T02:16:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-28T02:16:56,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732760213471 2024-11-28T02:16:56,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "hbase-failsafe-snapshot_backup_system-1732760213471" type: DISABLED 2024-11-28T02:16:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1732760213471 2024-11-28T02:16:56,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.2 enable backup:system 2024-11-28T02:16:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-11-28T02:16:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:16:56,288 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-11-28T02:16:56,288 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760216288"}]},"ts":"1732760216288"} 2024-11-28T02:16:56,289 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-28T02:16:56,290 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-11-28T02:16:56,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN}] 2024-11-28T02:16:56,291 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN 2024-11-28T02:16:56,291 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-11-28T02:16:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:16:56,442 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=a2d65c385b1b9e1c5b04dfa38326a47a, regionState=OPENING, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:16:56,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN because future has completed 2024-11-28T02:16:56,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598}] 2024-11-28T02:16:56,501 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 30e66b01c115b81cd680f3864aaf6f9d, had cached 0 bytes from a total of 13547 2024-11-28T02:16:56,599 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:56,599 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => a2d65c385b1b9e1c5b04dfa38326a47a, NAME => 'backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.', STARTKEY => '', ENDKEY => ''} 2024-11-28T02:16:56,600 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-28T02:16:56,600 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,600 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T02:16:56,600 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,600 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,601 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,602 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2d65c385b1b9e1c5b04dfa38326a47a columnFamilyName meta 2024-11-28T02:16:56,602 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:16:56,608 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20 2024-11-28T02:16:56,612 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935 2024-11-28T02:16:56,616 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17 2024-11-28T02:16:56,616 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.HStore(327): Store=a2d65c385b1b9e1c5b04dfa38326a47a/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:16:56,616 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,617 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2d65c385b1b9e1c5b04dfa38326a47a columnFamilyName session 2024-11-28T02:16:56,617 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T02:16:56,623 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba 2024-11-28T02:16:56,626 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a 2024-11-28T02:16:56,630 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 2024-11-28T02:16:56,634 DEBUG [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486 2024-11-28T02:16:56,634 INFO [StoreOpener-a2d65c385b1b9e1c5b04dfa38326a47a-1 {}] regionserver.HStore(327): Store=a2d65c385b1b9e1c5b04dfa38326a47a/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T02:16:56,634 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,635 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,636 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,637 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,637 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,637 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-28T02:16:56,638 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,639 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened a2d65c385b1b9e1c5b04dfa38326a47a; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10395849280, jitterRate=-0.03181108832359314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-28T02:16:56,639 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,640 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for a2d65c385b1b9e1c5b04dfa38326a47a: Running coprocessor pre-open hook at 1732760216600Writing region info on filesystem at 1732760216600Initializing all the Stores at 1732760216601 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760216601Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732760216601Cleaning up temporary data from old regions at 1732760216637 (+36 ms)Running coprocessor post-open hooks at 1732760216639 (+2 ms)Region opened successfully at 1732760216640 (+1 ms) 2024-11-28T02:16:56,641 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., pid=70, masterSystemTime=1732760216596 2024-11-28T02:16:56,642 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store a2d65c385b1b9e1c5b04dfa38326a47a:meta, priority=-2147483648, current under compaction store size is 1 2024-11-28T02:16:56,644 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:16:56,644 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store a2d65c385b1b9e1c5b04dfa38326a47a:session, priority=-2147483648, current under compaction store size is 2 2024-11-28T02:16:56,644 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T02:16:56,644 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T02:16:56,644 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T02:16:56,646 DEBUG [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:56,646 INFO [RS_OPEN_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:56,646 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T02:16:56,646 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T02:16:56,647 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=a2d65c385b1b9e1c5b04dfa38326a47a, regionState=OPEN, openSeqNum=51, regionLocation=d9827d9b919f,42127,1732760115598 2024-11-28T02:16:56,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598 because future has completed 2024-11-28T02:16:56,648 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.HStore(1541): a2d65c385b1b9e1c5b04dfa38326a47a/session is initiating minor compaction (all files) 2024-11-28T02:16:56,648 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.HStore(1541): a2d65c385b1b9e1c5b04dfa38326a47a/meta is initiating minor compaction (all files) 2024-11-28T02:16:56,649 INFO [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2d65c385b1b9e1c5b04dfa38326a47a/meta in backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:56,649 INFO [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2d65c385b1b9e1c5b04dfa38326a47a/session in backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:56,649 INFO [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20] into tmpdir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp, totalSize=17.9 K 2024-11-28T02:16:56,649 INFO [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7] into tmpdir=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp, totalSize=22.7 K 2024-11-28T02:16:56,650 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] compactions.Compactor(225): Compacting 44177ac9f4e946898d32e4854b5d17ba, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732760134001 2024-11-28T02:16:56,650 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] compactions.Compactor(225): Compacting f87bf723c61349d4b13e444028724d17, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732760136333 2024-11-28T02:16:56,651 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e26820e292547718ba9895fc2aefe8a, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732760149891 2024-11-28T02:16:56,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-28T02:16:56,651 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98da890d50746ecbbb10200c71bf935, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732760156007 2024-11-28T02:16:56,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure a2d65c385b1b9e1c5b04dfa38326a47a, server=d9827d9b919f,42127,1732760115598 in 205 msec 2024-11-28T02:16:56,651 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] compactions.Compactor(225): Compacting f4071ad0b88840998cb64d2c91a76486, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732760164463 2024-11-28T02:16:56,652 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] compactions.Compactor(225): Compacting 285993cd51774e86a2bc07a5f816cc20, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732760169820 2024-11-28T02:16:56,652 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] compactions.Compactor(225): Compacting e1d6100cf32c4f1f9ab5c7b5c69472f7, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732760196319 2024-11-28T02:16:56,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-28T02:16:56,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=a2d65c385b1b9e1c5b04dfa38326a47a, ASSIGN in 361 msec 2024-11-28T02:16:56,656 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732760216656"}]},"ts":"1732760216656"} 2024-11-28T02:16:56,658 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-28T02:16:56,658 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-11-28T02:16:56,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 377 msec 2024-11-28T02:16:56,685 INFO [RS:0;d9827d9b919f:42127-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2d65c385b1b9e1c5b04dfa38326a47a#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:16:56,685 INFO [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2d65c385b1b9e1c5b04dfa38326a47a#meta#compaction#16 average throughput is 1.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T02:16:56,686 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/24f7028acbea4c2ab845d0c9a157d845 is 1245, key is session:backup_1732760165447/session:context/1732760165534/Put/seqid=0 2024-11-28T02:16:56,686 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/4335ae8616e94d679c47d99024498e03 is 158, key is trslm:hdfs://localhost:38631/backupUT\x00ns2:test-17327601242691/meta:log-roll-map/1732760195885/Put/seqid=0 2024-11-28T02:16:56,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741974_1150 (size=8073) 2024-11-28T02:16:56,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741975_1151 (size=6749) 2024-11-28T02:16:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T02:16:56,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-11-28T02:16:56,914 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-11-28T02:16:56,914 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-28T02:16:56,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-28T02:16:56,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43413 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-28T02:16:56,921 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-28T02:16:56,921 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-11-28T02:16:56,923 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-28T02:16:56,927 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732760201348 failed. 2024-11-28T02:16:56,927 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:16:56,927 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:56,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:56,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:56,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:56,950 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=627 (was 611) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1940498200_20 at /127.0.0.1:43484 [Receiving block BP-903783156-172.17.0.2-1732760110915:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-903783156-172.17.0.2-1732760110915:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (d9827d9b919f,43413,1732760114080)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: 'd9827d9b919f,42127,1732760115598' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-903783156-172.17.0.2-1732760110915:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1940498200_20 at /127.0.0.1:43494 [Receiving block BP-903783156-172.17.0.2-1732760110915:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/4335ae8616e94d679c47d99024498e03 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-591416287_20 at /127.0.0.1:53674 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(d9827d9b919f,42127,1732760115598)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/d9827d9b919f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/24f7028acbea4c2ab845d0c9a157d845 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;d9827d9b919f:42127-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1940498200_20 at /127.0.0.1:43508 [Receiving block BP-903783156-172.17.0.2-1732760110915:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-903783156-172.17.0.2-1732760110915:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=639 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=68 (was 107), ProcessCount=14 (was 14), AvailableMemoryMB=7408 (was 7856) 2024-11-28T02:16:56,952 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=627 is superior to 500 2024-11-28T02:16:56,958 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T02:16:56,959 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:16:56,959 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:56,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:56,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:56,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:56,959 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T02:16:56,959 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1486845682, stopped=false 2024-11-28T02:16:56,959 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d9827d9b919f,43413,1732760114080 2024-11-28T02:16:56,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-28T02:16:56,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-28T02:16:56,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:16:56,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:16:56,961 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T02:16:56,961 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T02:16:56,962 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:56,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:56,962 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-28T02:16:56,962 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-28T02:16:56,962 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9827d9b919f,42127,1732760115598' ***** 2024-11-28T02:16:56,963 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T02:16:56,963 INFO [RS:0;d9827d9b919f:42127 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-28T02:16:56,967 INFO [RS:0;d9827d9b919f:42127 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@9984275{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-28T02:16:56,969 INFO [RS:0;d9827d9b919f:42127 {}] server.AbstractConnector(383): Stopped ServerConnector@445a8045{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-28T02:16:56,969 INFO [RS:0;d9827d9b919f:42127 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:16:56,969 INFO [RS:0;d9827d9b919f:42127 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2df3a35c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-28T02:16:56,969 INFO [RS:0;d9827d9b919f:42127 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3e9615b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,STOPPED} 2024-11-28T02:16:56,971 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T02:16:56,971 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T02:16:56,971 INFO [RS:0;d9827d9b919f:42127 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T02:16:56,971 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-28T02:16:56,971 INFO [RS:0;d9827d9b919f:42127 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(3091): Received CLOSE for a2d65c385b1b9e1c5b04dfa38326a47a 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(3091): Received CLOSE for 30e66b01c115b81cd680f3864aaf6f9d 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(3091): Received CLOSE for df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(3091): Received CLOSE for d29139dea3e9e00e0b5beeb2ae62089f 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(3091): Received CLOSE for 63374509f749d336450706661d23059d 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(3091): Received CLOSE for 795ba51d3cf9aa45e08a6768f6136859 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(959): stopping server d9827d9b919f,42127,1732760115598 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T02:16:56,972 INFO [RS:0;d9827d9b919f:42127 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d9827d9b919f:42127. 2024-11-28T02:16:56,972 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a2d65c385b1b9e1c5b04dfa38326a47a, disabling compactions & flushes 2024-11-28T02:16:56,972 DEBUG [RS:0;d9827d9b919f:42127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:56,972 DEBUG [RS:0;d9827d9b919f:42127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:56,972 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:56,973 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T02:16:57,015 INFO [regionserver/d9827d9b919f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T02:16:57,098 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/24f7028acbea4c2ab845d0c9a157d845 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/24f7028acbea4c2ab845d0c9a157d845 2024-11-28T02:16:57,098 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/meta/4335ae8616e94d679c47d99024498e03 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/4335ae8616e94d679c47d99024498e03 2024-11-28T02:16:57,109 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-28T02:16:57,109 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-28T02:16:57,111 INFO [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2d65c385b1b9e1c5b04dfa38326a47a/session of a2d65c385b1b9e1c5b04dfa38326a47a into 24f7028acbea4c2ab845d0c9a157d845(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:16:57,111 INFO [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2d65c385b1b9e1c5b04dfa38326a47a/meta of a2d65c385b1b9e1c5b04dfa38326a47a into 4335ae8616e94d679c47d99024498e03(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T02:16:57,111 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2d65c385b1b9e1c5b04dfa38326a47a: 2024-11-28T02:16:57,111 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2d65c385b1b9e1c5b04dfa38326a47a: 2024-11-28T02:16:57,111 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:57,111 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:57,111 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. after waiting 0 ms 2024-11-28T02:16:57,111 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:57,112 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a2d65c385b1b9e1c5b04dfa38326a47a 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-11-28T02:16:57,113 INFO [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., storeName=a2d65c385b1b9e1c5b04dfa38326a47a/session, priority=12, startTime=1732760216644; duration=0sec 2024-11-28T02:16:57,113 INFO [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., storeName=a2d65c385b1b9e1c5b04dfa38326a47a/meta, priority=13, startTime=1732760216642; duration=0sec 2024-11-28T02:16:57,114 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:16:57,114 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T02:16:57,114 DEBUG [RS:0;d9827d9b919f:42127-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2d65c385b1b9e1c5b04dfa38326a47a:session 2024-11-28T02:16:57,114 DEBUG [RS:0;d9827d9b919f:42127-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2d65c385b1b9e1c5b04dfa38326a47a:meta 2024-11-28T02:16:57,114 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T02:16:57,114 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T02:16:57,114 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T02:16:57,115 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-11-28T02:16:57,115 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T02:16:57,115 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1325): Online Regions={a2d65c385b1b9e1c5b04dfa38326a47a=backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., 30e66b01c115b81cd680f3864aaf6f9d=ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d., df85333d0693eea3c91e4c0b6b737054=test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., d29139dea3e9e00e0b5beeb2ae62089f=ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f., 1588230740=hbase:meta,,1.1588230740, 63374509f749d336450706661d23059d=backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d., 795ba51d3cf9aa45e08a6768f6136859=ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859.} 2024-11-28T02:16:57,115 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T02:16:57,115 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T02:16:57,115 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T02:16:57,115 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T02:16:57,115 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-11-28T02:16:57,115 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 30e66b01c115b81cd680f3864aaf6f9d, 63374509f749d336450706661d23059d, 795ba51d3cf9aa45e08a6768f6136859, a2d65c385b1b9e1c5b04dfa38326a47a, d29139dea3e9e00e0b5beeb2ae62089f, df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:57,116 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/ad6979d66678422b8ab64d8fe9b3bd06 is 554, key is session:backup_1732760201348/session:context/1732760216922/Put/seqid=0 2024-11-28T02:16:57,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741976_1152 (size=5655) 2024-11-28T02:16:57,142 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/info/c582552f800640e89fb62315e924e59d is 159, key is ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d./info:regioninfo/1732760126524/Put/seqid=0 2024-11-28T02:16:57,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741977_1153 (size=14126) 2024-11-28T02:16:57,226 INFO [regionserver/d9827d9b919f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T02:16:57,226 INFO [regionserver/d9827d9b919f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T02:16:57,315 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 30e66b01c115b81cd680f3864aaf6f9d, 63374509f749d336450706661d23059d, 795ba51d3cf9aa45e08a6768f6136859, a2d65c385b1b9e1c5b04dfa38326a47a, d29139dea3e9e00e0b5beeb2ae62089f, df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:57,515 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 30e66b01c115b81cd680f3864aaf6f9d, 63374509f749d336450706661d23059d, 795ba51d3cf9aa45e08a6768f6136859, a2d65c385b1b9e1c5b04dfa38326a47a, d29139dea3e9e00e0b5beeb2ae62089f, df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:57,520 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/ad6979d66678422b8ab64d8fe9b3bd06 2024-11-28T02:16:57,526 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/.tmp/session/ad6979d66678422b8ab64d8fe9b3bd06 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/ad6979d66678422b8ab64d8fe9b3bd06 2024-11-28T02:16:57,531 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/ad6979d66678422b8ab64d8fe9b3bd06, entries=2, sequenceid=57, filesize=5.5 K 2024-11-28T02:16:57,532 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for a2d65c385b1b9e1c5b04dfa38326a47a in 421ms, sequenceid=57, compaction requested=false 2024-11-28T02:16:57,533 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20] to archive 2024-11-28T02:16:57,533 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:16:57,536 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/f87bf723c61349d4b13e444028724d17 2024-11-28T02:16:57,538 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/c98da890d50746ecbbb10200c71bf935 2024-11-28T02:16:57,539 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/meta/285993cd51774e86a2bc07a5f816cc20 2024-11-28T02:16:57,544 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d9827d9b919f:43413 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-28T02:16:57,545 WARN [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [f87bf723c61349d4b13e444028724d17=5914, c98da890d50746ecbbb10200c71bf935=5745, 285993cd51774e86a2bc07a5f816cc20=6646] 2024-11-28T02:16:57,545 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7] to archive 2024-11-28T02:16:57,546 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T02:16:57,547 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/info/c582552f800640e89fb62315e924e59d 2024-11-28T02:16:57,548 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/44177ac9f4e946898d32e4854b5d17ba 2024-11-28T02:16:57,549 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/5e26820e292547718ba9895fc2aefe8a 2024-11-28T02:16:57,550 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/f4071ad0b88840998cb64d2c91a76486 2024-11-28T02:16:57,552 DEBUG [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 to hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/archive/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/session/e1d6100cf32c4f1f9ab5c7b5c69472f7 2024-11-28T02:16:57,552 WARN [StoreCloser-backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [44177ac9f4e946898d32e4854b5d17ba=5049, 5e26820e292547718ba9895fc2aefe8a=5535, f4071ad0b88840998cb64d2c91a76486=6682, e1d6100cf32c4f1f9ab5c7b5c69472f7=5928] 2024-11-28T02:16:57,555 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system/a2d65c385b1b9e1c5b04dfa38326a47a/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-11-28T02:16:57,555 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:57,556 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:57,556 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a2d65c385b1b9e1c5b04dfa38326a47a: Waiting for close lock at 1732760216972Running coprocessor pre-close hooks at 1732760216972Disabling compacts and flushes for region at 1732760216972Disabling writes for close at 1732760217111 (+139 ms)Obtaining lock to block concurrent updates at 1732760217112 (+1 ms)Preparing flush snapshotting stores in a2d65c385b1b9e1c5b04dfa38326a47a at 1732760217112Finished memstore snapshotting backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1732760217112Flushing stores of backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. at 1732760217112Flushing a2d65c385b1b9e1c5b04dfa38326a47a/session: creating writer at 1732760217112Flushing a2d65c385b1b9e1c5b04dfa38326a47a/session: appending metadata at 1732760217115 (+3 ms)Flushing a2d65c385b1b9e1c5b04dfa38326a47a/session: closing flushed file at 1732760217115Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a502fbc: reopening flushed file at 1732760217526 (+411 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for a2d65c385b1b9e1c5b04dfa38326a47a in 421ms, sequenceid=57, compaction requested=false at 1732760217532 (+6 ms)Writing region close event to WAL at 1732760217552 (+20 ms)Running coprocessor post-close hooks at 1732760217555 (+3 ms)Closed at 1732760217556 (+1 ms) 2024-11-28T02:16:57,556 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1732760132681.a2d65c385b1b9e1c5b04dfa38326a47a. 2024-11-28T02:16:57,556 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30e66b01c115b81cd680f3864aaf6f9d, disabling compactions & flushes 2024-11-28T02:16:57,556 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:16:57,556 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:16:57,556 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. after waiting 0 ms 2024-11-28T02:16:57,556 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:16:57,559 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns2/test-17327601242691/30e66b01c115b81cd680f3864aaf6f9d/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-28T02:16:57,560 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:57,560 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:16:57,560 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30e66b01c115b81cd680f3864aaf6f9d: Waiting for close lock at 1732760217556Running coprocessor pre-close hooks at 1732760217556Disabling compacts and flushes for region at 1732760217556Disabling writes for close at 1732760217556Writing region close event to WAL at 1732760217557 (+1 ms)Running coprocessor post-close hooks at 1732760217560 (+3 ms)Closed at 1732760217560 2024-11-28T02:16:57,560 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17327601242691,,1732760126150.30e66b01c115b81cd680f3864aaf6f9d. 2024-11-28T02:16:57,560 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df85333d0693eea3c91e4c0b6b737054, disabling compactions & flushes 2024-11-28T02:16:57,560 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:57,560 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:57,560 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. after waiting 0 ms 2024-11-28T02:16:57,560 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:57,560 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing df85333d0693eea3c91e4c0b6b737054 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-28T02:16:57,568 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/ns/52677b64dc0641e2a09e3fd1d9c1bd58 is 43, key is default/ns:d/1732760119321/Put/seqid=0 2024-11-28T02:16:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741978_1154 (size=5345) 2024-11-28T02:16:57,579 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f2/f895c8daeb4540a88b3839a7296c01c7 is 58, key is row-test-1732760124269-2-0/f2:q1/1732760196340/Put/seqid=0 2024-11-28T02:16:57,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741979_1155 (size=5464) 2024-11-28T02:16:57,716 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 63374509f749d336450706661d23059d, 795ba51d3cf9aa45e08a6768f6136859, d29139dea3e9e00e0b5beeb2ae62089f, df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:57,916 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 63374509f749d336450706661d23059d, 795ba51d3cf9aa45e08a6768f6136859, d29139dea3e9e00e0b5beeb2ae62089f, df85333d0693eea3c91e4c0b6b737054 2024-11-28T02:16:57,975 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/ns/52677b64dc0641e2a09e3fd1d9c1bd58 2024-11-28T02:16:57,983 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f2/f895c8daeb4540a88b3839a7296c01c7 2024-11-28T02:16:57,989 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/.tmp/f2/f895c8daeb4540a88b3839a7296c01c7 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f2/f895c8daeb4540a88b3839a7296c01c7 2024-11-28T02:16:57,993 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/f2/f895c8daeb4540a88b3839a7296c01c7, entries=7, sequenceid=348, filesize=5.3 K 2024-11-28T02:16:57,994 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for df85333d0693eea3c91e4c0b6b737054 in 434ms, sequenceid=348, compaction requested=false 2024-11-28T02:16:57,995 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/table/7ca33e603f8d411babbc713f0f56da19 is 59, key is ns2:test-17327601242691/table:state/1732760126538/Put/seqid=0 2024-11-28T02:16:57,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741980_1156 (size=5866) 2024-11-28T02:16:57,999 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/default/test-1732760124269/df85333d0693eea3c91e4c0b6b737054/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-28T02:16:58,000 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:58,000 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:58,000 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df85333d0693eea3c91e4c0b6b737054: Waiting for close lock at 1732760217560Running coprocessor pre-close hooks at 1732760217560Disabling compacts and flushes for region at 1732760217560Disabling writes for close at 1732760217560Obtaining lock to block concurrent updates at 1732760217560Preparing flush snapshotting stores in df85333d0693eea3c91e4c0b6b737054 at 1732760217560Finished memstore snapshotting test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1732760217561 (+1 ms)Flushing stores of test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. at 1732760217561Flushing df85333d0693eea3c91e4c0b6b737054/f2: creating writer at 1732760217561Flushing df85333d0693eea3c91e4c0b6b737054/f2: appending metadata at 1732760217579 (+18 ms)Flushing df85333d0693eea3c91e4c0b6b737054/f2: closing flushed file at 1732760217579Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fcf3f36: reopening flushed file at 1732760217988 (+409 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for df85333d0693eea3c91e4c0b6b737054 in 434ms, sequenceid=348, compaction requested=false at 1732760217994 (+6 ms)Writing region close event to WAL at 1732760217996 (+2 ms)Running coprocessor post-close hooks at 1732760218000 (+4 ms)Closed at 1732760218000 2024-11-28T02:16:58,000 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1732760124269,,1732760124779.df85333d0693eea3c91e4c0b6b737054. 2024-11-28T02:16:58,000 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d29139dea3e9e00e0b5beeb2ae62089f, disabling compactions & flushes 2024-11-28T02:16:58,000 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:16:58,000 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:16:58,000 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. after waiting 0 ms 2024-11-28T02:16:58,000 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:16:58,003 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns4/test-17327601242693/d29139dea3e9e00e0b5beeb2ae62089f/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-28T02:16:58,003 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:58,003 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:16:58,003 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d29139dea3e9e00e0b5beeb2ae62089f: Waiting for close lock at 1732760218000Running coprocessor pre-close hooks at 1732760218000Disabling compacts and flushes for region at 1732760218000Disabling writes for close at 1732760218000Writing region close event to WAL at 1732760218000Running coprocessor post-close hooks at 1732760218003 (+3 ms)Closed at 1732760218003 2024-11-28T02:16:58,004 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17327601242693,,1732760129094.d29139dea3e9e00e0b5beeb2ae62089f. 2024-11-28T02:16:58,004 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 63374509f749d336450706661d23059d, disabling compactions & flushes 2024-11-28T02:16:58,004 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:16:58,004 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:16:58,004 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. after waiting 0 ms 2024-11-28T02:16:58,004 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:16:58,006 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/backup/system_bulk/63374509f749d336450706661d23059d/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-28T02:16:58,007 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:58,007 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:16:58,007 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 63374509f749d336450706661d23059d: Waiting for close lock at 1732760218004Running coprocessor pre-close hooks at 1732760218004Disabling compacts and flushes for region at 1732760218004Disabling writes for close at 1732760218004Writing region close event to WAL at 1732760218004Running coprocessor post-close hooks at 1732760218007 (+3 ms)Closed at 1732760218007 2024-11-28T02:16:58,007 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1732760133332.63374509f749d336450706661d23059d. 2024-11-28T02:16:58,007 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 795ba51d3cf9aa45e08a6768f6136859, disabling compactions & flushes 2024-11-28T02:16:58,007 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:16:58,007 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:16:58,007 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. after waiting 0 ms 2024-11-28T02:16:58,007 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:16:58,010 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/ns3/test-17327601242692/795ba51d3cf9aa45e08a6768f6136859/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-28T02:16:58,010 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:58,010 INFO [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:16:58,010 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 795ba51d3cf9aa45e08a6768f6136859: Waiting for close lock at 1732760218007Running coprocessor pre-close hooks at 1732760218007Disabling compacts and flushes for region at 1732760218007Disabling writes for close at 1732760218007Writing region close event to WAL at 1732760218007Running coprocessor post-close hooks at 1732760218010 (+3 ms)Closed at 1732760218010 2024-11-28T02:16:58,010 DEBUG [RS_CLOSE_REGION-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17327601242692,,1732760126923.795ba51d3cf9aa45e08a6768f6136859. 2024-11-28T02:16:58,116 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T02:16:58,116 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T02:16:58,116 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:16:58,316 DEBUG [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T02:16:58,400 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/table/7ca33e603f8d411babbc713f0f56da19 2024-11-28T02:16:58,406 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/info/c582552f800640e89fb62315e924e59d as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/info/c582552f800640e89fb62315e924e59d 2024-11-28T02:16:58,410 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/info/c582552f800640e89fb62315e924e59d, entries=69, sequenceid=59, filesize=13.8 K 2024-11-28T02:16:58,411 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/ns/52677b64dc0641e2a09e3fd1d9c1bd58 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/ns/52677b64dc0641e2a09e3fd1d9c1bd58 2024-11-28T02:16:58,415 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/ns/52677b64dc0641e2a09e3fd1d9c1bd58, entries=7, sequenceid=59, filesize=5.2 K 2024-11-28T02:16:58,416 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/.tmp/table/7ca33e603f8d411babbc713f0f56da19 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/table/7ca33e603f8d411babbc713f0f56da19 2024-11-28T02:16:58,421 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/table/7ca33e603f8d411babbc713f0f56da19, entries=13, sequenceid=59, filesize=5.7 K 2024-11-28T02:16:58,422 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1306ms, sequenceid=59, compaction requested=false 2024-11-28T02:16:58,426 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-11-28T02:16:58,427 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-28T02:16:58,427 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T02:16:58,427 INFO [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T02:16:58,427 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732760217115Running coprocessor pre-close hooks at 1732760217115Disabling compacts and flushes for region at 1732760217115Disabling writes for close at 1732760217115Obtaining lock to block concurrent updates at 1732760217115Preparing flush snapshotting stores in 1588230740 at 1732760217115Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1732760217115Flushing stores of hbase:meta,,1.1588230740 at 1732760217116 (+1 ms)Flushing 1588230740/info: creating writer at 1732760217116Flushing 1588230740/info: appending metadata at 1732760217141 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732760217142 (+1 ms)Flushing 1588230740/ns: creating writer at 1732760217552 (+410 ms)Flushing 1588230740/ns: appending metadata at 1732760217567 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732760217567Flushing 1588230740/table: creating writer at 1732760217980 (+413 ms)Flushing 1588230740/table: appending metadata at 1732760217994 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732760217994Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b717a37: reopening flushed file at 1732760218405 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@82ee4ff: reopening flushed file at 1732760218410 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68b63fb5: reopening flushed file at 1732760218415 (+5 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1306ms, sequenceid=59, compaction requested=false at 1732760218422 (+7 ms)Writing region close event to WAL at 1732760218423 (+1 ms)Running coprocessor post-close hooks at 1732760218426 (+3 ms)Closed at 1732760218427 (+1 ms) 2024-11-28T02:16:58,427 DEBUG [RS_CLOSE_META-regionserver/d9827d9b919f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T02:16:58,517 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(976): stopping server d9827d9b919f,42127,1732760115598; all regions closed. 2024-11-28T02:16:58,520 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.meta.1732760205678.meta not finished, retry = 0 2024-11-28T02:16:58,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741954_1130 (size=3260) 2024-11-28T02:16:58,625 DEBUG [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs 2024-11-28T02:16:58,625 INFO [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.meta:.meta(num 1732760205678) 2024-11-28T02:16:58,627 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/WALs/d9827d9b919f,42127,1732760115598/d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0.1732760205711 not finished, retry = 0 2024-11-28T02:16:58,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741956_1132 (size=1037) 2024-11-28T02:16:58,730 DEBUG [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs 2024-11-28T02:16:58,730 INFO [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-0:(num 1732760205711) 2024-11-28T02:16:58,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741955_1131 (size=5275) 2024-11-28T02:16:58,734 DEBUG [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/oldWALs 2024-11-28T02:16:58,734 INFO [RS:0;d9827d9b919f:42127 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d9827d9b919f%2C42127%2C1732760115598.d9827d9b919f%2C42127%2C1732760115598.regiongroup-1:(num 1732760205695) 2024-11-28T02:16:58,734 DEBUG [RS:0;d9827d9b919f:42127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:58,734 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T02:16:58,735 INFO [RS:0;d9827d9b919f:42127 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T02:16:58,735 INFO [RS:0;d9827d9b919f:42127 {}] hbase.ChoreService(370): Chore service for: regionserver/d9827d9b919f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T02:16:58,735 INFO [RS:0;d9827d9b919f:42127 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T02:16:58,735 INFO [regionserver/d9827d9b919f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T02:16:58,736 INFO [RS:0;d9827d9b919f:42127 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42127 2024-11-28T02:16:58,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/d9827d9b919f,42127,1732760115598 2024-11-28T02:16:58,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-28T02:16:58,740 INFO [RS:0;d9827d9b919f:42127 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T02:16:58,742 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9827d9b919f,42127,1732760115598] 2024-11-28T02:16:58,745 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/d9827d9b919f,42127,1732760115598 already deleted, retry=false 2024-11-28T02:16:58,745 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9827d9b919f,42127,1732760115598 expired; onlineServers=0 2024-11-28T02:16:58,745 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd9827d9b919f,43413,1732760114080' ***** 2024-11-28T02:16:58,745 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T02:16:58,745 INFO [M:0;d9827d9b919f:43413 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-28T02:16:58,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-28T02:16:58,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-28T02:16:58,754 INFO [M:0;d9827d9b919f:43413 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5c685ba8{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-28T02:16:58,754 INFO [M:0;d9827d9b919f:43413 {}] server.AbstractConnector(383): Stopped ServerConnector@10f9ba48{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-28T02:16:58,754 INFO [M:0;d9827d9b919f:43413 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:16:58,754 INFO [M:0;d9827d9b919f:43413 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@65507143{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-28T02:16:58,754 INFO [M:0;d9827d9b919f:43413 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7c1597dd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,STOPPED} 2024-11-28T02:16:58,755 INFO [M:0;d9827d9b919f:43413 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T02:16:58,755 INFO [M:0;d9827d9b919f:43413 {}] master.HMaster(1770): Stopping master jetty server 2024-11-28T02:16:58,756 INFO [M:0;d9827d9b919f:43413 {}] server.AbstractConnector(383): Stopped ServerConnector@106565e3{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-28T02:16:58,756 INFO [M:0;d9827d9b919f:43413 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T02:16:58,756 DEBUG [M:0;d9827d9b919f:43413 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T02:16:58,756 INFO [M:0;d9827d9b919f:43413 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-28T02:16:58,756 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T02:16:58,756 INFO [M:0;d9827d9b919f:43413 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;d9827d9b919f:43413. 2024-11-28T02:16:58,757 DEBUG [master/d9827d9b919f:0:becomeActiveMaster-HFileCleaner.large.0-1732760117957 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9827d9b919f:0:becomeActiveMaster-HFileCleaner.large.0-1732760117957,5,FailOnTimeoutGroup] 2024-11-28T02:16:58,757 DEBUG [master/d9827d9b919f:0:becomeActiveMaster-HFileCleaner.small.0-1732760117958 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9827d9b919f:0:becomeActiveMaster-HFileCleaner.small.0-1732760117958,5,FailOnTimeoutGroup] 2024-11-28T02:16:58,757 DEBUG [M:0;d9827d9b919f:43413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T02:16:58,757 DEBUG [M:0;d9827d9b919f:43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:58,757 DEBUG [M:0;d9827d9b919f:43413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T02:16:58,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T02:16:58,757 DEBUG [M:0;d9827d9b919f:43413 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T02:16:58,757 INFO [M:0;d9827d9b919f:43413 {}] hbase.ChoreService(370): Chore service for: master/d9827d9b919f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T02:16:58,757 INFO [M:0;d9827d9b919f:43413 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T02:16:58,757 DEBUG [M:0;d9827d9b919f:43413 {}] master.HMaster(1795): Stopping service threads 2024-11-28T02:16:58,757 INFO [M:0;d9827d9b919f:43413 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T02:16:58,757 INFO [M:0;d9827d9b919f:43413 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T02:16:58,758 INFO [M:0;d9827d9b919f:43413 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T02:16:58,758 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T02:16:58,759 DEBUG [M:0;d9827d9b919f:43413 {}] zookeeper.ZKUtil(347): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-28T02:16:58,759 WARN [M:0;d9827d9b919f:43413 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T02:16:58,759 INFO [M:0;d9827d9b919f:43413 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/.lastflushedseqids 2024-11-28T02:16:58,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741981_1157 (size=495) 2024-11-28T02:16:58,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T02:16:58,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42127-0x1015c5afacc0001, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T02:16:58,843 INFO [RS:0;d9827d9b919f:42127 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T02:16:58,843 INFO [RS:0;d9827d9b919f:42127 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9827d9b919f,42127,1732760115598; zookeeper connection closed. 2024-11-28T02:16:58,844 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@249c1760 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@249c1760 2024-11-28T02:16:58,844 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T02:16:58,851 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-28T02:16:59,174 INFO [M:0;d9827d9b919f:43413 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T02:16:59,174 INFO [M:0;d9827d9b919f:43413 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T02:16:59,174 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T02:16:59,188 INFO [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:16:59,188 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:16:59,188 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T02:16:59,188 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:16:59,188 INFO [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=319.26 KB heapSize=375.38 KB 2024-11-28T02:16:59,204 DEBUG [M:0;d9827d9b919f:43413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0f1cbca4e1e42449a477f0e6c81154a is 82, key is hbase:meta,,1/info:regioninfo/1732760119251/Put/seqid=0 2024-11-28T02:16:59,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741982_1158 (size=5672) 2024-11-28T02:16:59,608 INFO [M:0;d9827d9b919f:43413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0f1cbca4e1e42449a477f0e6c81154a 2024-11-28T02:16:59,629 DEBUG [M:0;d9827d9b919f:43413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e3e96bd0f8244f999d7526a7800c41b5 is 2184, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1732760165445/Put/seqid=0 2024-11-28T02:16:59,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741983_1159 (size=26519) 2024-11-28T02:17:00,034 INFO [M:0;d9827d9b919f:43413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.70 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e3e96bd0f8244f999d7526a7800c41b5 2024-11-28T02:17:00,039 INFO [M:0;d9827d9b919f:43413 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e3e96bd0f8244f999d7526a7800c41b5 2024-11-28T02:17:00,054 DEBUG [M:0;d9827d9b919f:43413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4d66f8f7e49f4a3bb02cab6ec692ae84 is 69, key is d9827d9b919f,42127,1732760115598/rs:state/1732760118069/Put/seqid=0 2024-11-28T02:17:00,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741984_1160 (size=5156) 2024-11-28T02:17:00,458 INFO [M:0;d9827d9b919f:43413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4d66f8f7e49f4a3bb02cab6ec692ae84 2024-11-28T02:17:00,464 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c0f1cbca4e1e42449a477f0e6c81154a as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c0f1cbca4e1e42449a477f0e6c81154a 2024-11-28T02:17:00,468 INFO [M:0;d9827d9b919f:43413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c0f1cbca4e1e42449a477f0e6c81154a, entries=8, sequenceid=667, filesize=5.5 K 2024-11-28T02:17:00,469 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e3e96bd0f8244f999d7526a7800c41b5 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e3e96bd0f8244f999d7526a7800c41b5 2024-11-28T02:17:00,473 INFO [M:0;d9827d9b919f:43413 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e3e96bd0f8244f999d7526a7800c41b5 2024-11-28T02:17:00,473 INFO [M:0;d9827d9b919f:43413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e3e96bd0f8244f999d7526a7800c41b5, entries=69, sequenceid=667, filesize=25.9 K 2024-11-28T02:17:00,474 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4d66f8f7e49f4a3bb02cab6ec692ae84 as hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4d66f8f7e49f4a3bb02cab6ec692ae84 2024-11-28T02:17:00,478 INFO [M:0;d9827d9b919f:43413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38631/user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4d66f8f7e49f4a3bb02cab6ec692ae84, entries=1, sequenceid=667, filesize=5.0 K 2024-11-28T02:17:00,479 INFO [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(3140): Finished flush of dataSize ~319.26 KB/326921, heapSize ~375.09 KB/384088, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=667, compaction requested=false 2024-11-28T02:17:00,480 INFO [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T02:17:00,480 DEBUG [M:0;d9827d9b919f:43413 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732760219174Disabling compacts and flushes for region at 1732760219174Disabling writes for close at 1732760219188 (+14 ms)Obtaining lock to block concurrent updates at 1732760219188Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732760219188Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326921, getHeapSize=384328, getOffHeapSize=0, getCellsCount=746 at 1732760219189 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732760219189Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732760219189Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732760219203 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732760219203Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732760219613 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732760219629 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732760219629Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732760220039 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732760220053 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732760220053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c0f5ba9: reopening flushed file at 1732760220463 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58d93c83: reopening flushed file at 1732760220468 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5171d116: reopening flushed file at 1732760220473 (+5 ms)Finished flush of dataSize ~319.26 KB/326921, heapSize ~375.09 KB/384088, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=667, compaction requested=false at 1732760220479 (+6 ms)Writing region close event to WAL at 1732760220480 (+1 ms)Closed at 1732760220480 2024-11-28T02:17:00,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37989 is added to blk_1073741830_1006 (size=371011) 2024-11-28T02:17:00,483 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a0eefd02-ddcc-6cf6-8a16-c3531026df96/MasterData/WALs/d9827d9b919f,43413,1732760114080/d9827d9b919f%2C43413%2C1732760114080.d9827d9b919f%2C43413%2C1732760114080.regiongroup-0.1732760117304 not finished, retry = 0 2024-11-28T02:17:00,584 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T02:17:00,584 INFO [M:0;d9827d9b919f:43413 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T02:17:00,584 INFO [M:0;d9827d9b919f:43413 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-28T02:17:00,585 INFO [M:0;d9827d9b919f:43413 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43413 2024-11-28T02:17:00,585 INFO [M:0;d9827d9b919f:43413 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T02:17:00,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T02:17:00,687 INFO [M:0;d9827d9b919f:43413 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T02:17:00,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43413-0x1015c5afacc0000, quorum=127.0.0.1:50908, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T02:17:00,697 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@cb44ad4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T02:17:00,699 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T02:17:00,699 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:17:00,699 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T02:17:00,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,STOPPED} 2024-11-28T02:17:00,702 WARN [BP-903783156-172.17.0.2-1732760110915 heartbeating to localhost/127.0.0.1:38631 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T02:17:00,702 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T02:17:00,702 WARN [BP-903783156-172.17.0.2-1732760110915 heartbeating to localhost/127.0.0.1:38631 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-903783156-172.17.0.2-1732760110915 (Datanode Uuid 96d72808-c4db-44d7-90dc-7b59517a8479) service to localhost/127.0.0.1:38631 2024-11-28T02:17:00,702 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T02:17:00,703 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/data/data1/current/BP-903783156-172.17.0.2-1732760110915 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T02:17:00,704 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/cluster_610555c7-9da5-66b6-d93b-8bfa8def81cd/data/data2/current/BP-903783156-172.17.0.2-1732760110915 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T02:17:00,704 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T02:17:00,712 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@38e78c7b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T02:17:00,713 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c336369{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T02:17:00,713 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:17:00,713 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b22dca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T02:17:00,713 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f045f3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,STOPPED} 2024-11-28T02:17:00,722 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T02:17:00,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-28T02:17:00,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-28T02:17:00,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@559a435d{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-28T02:17:00,777 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5f0f2e7d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T02:17:00,777 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:17:00,777 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74e2cc15{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-28T02:17:00,777 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@53df72c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,STOPPED} 2024-11-28T02:17:00,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55049bf1{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-28T02:17:00,790 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@534a2f07{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T02:17:00,791 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:17:00,791 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26a51899{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-28T02:17:00,791 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9e11c64{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,STOPPED} 2024-11-28T02:17:00,803 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-28T02:17:00,804 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bdc42df{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-28T02:17:00,804 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@567287a2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T02:17:00,804 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:17:00,804 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31795375{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-28T02:17:00,805 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ccdffea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,STOPPED} 2024-11-28T02:17:00,808 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-28T02:17:00,811 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-28T02:17:00,811 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-28T02:17:00,812 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-28T02:17:00,814 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@31ab3c62{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-28T02:17:00,816 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15a81a3d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T02:17:00,816 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T02:17:00,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a60e5b2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-28T02:17:00,817 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@171eacd3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/fb0cc34a-1a8d-d0f1-57e0-243b29da9098/hadoop.log.dir/,STOPPED} 2024-11-28T02:17:00,819 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-28T02:17:00,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped