2024-12-01 20:24:10,757 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-12-01 20:24:10,770 main DEBUG Took 0.010272 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-01 20:24:10,770 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-01 20:24:10,770 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-01 20:24:10,771 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-01 20:24:10,772 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,782 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-01 20:24:10,795 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,797 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,797 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,799 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,799 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,800 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,801 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,802 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,803 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,803 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,804 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,805 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:24:10,805 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,805 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-01 20:24:10,807 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:24:10,808 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-01 20:24:10,810 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-01 20:24:10,810 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-01 20:24:10,811 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-01 20:24:10,812 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-01 20:24:10,821 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-01 20:24:10,824 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-01 20:24:10,826 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-01 20:24:10,826 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-01 20:24:10,827 main DEBUG createAppenders(={Console}) 2024-12-01 20:24:10,828 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-12-01 20:24:10,828 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-12-01 20:24:10,828 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-12-01 20:24:10,829 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-01 20:24:10,829 main DEBUG OutputStream closed 2024-12-01 20:24:10,829 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-01 20:24:10,830 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-01 20:24:10,830 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-12-01 20:24:10,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-01 20:24:10,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-01 20:24:10,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-01 20:24:10,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-01 20:24:10,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-01 20:24:10,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-01 20:24:10,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-01 20:24:10,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-01 20:24:10,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-01 20:24:10,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-01 20:24:10,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-01 20:24:10,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-01 20:24:10,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-01 20:24:10,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-01 20:24:10,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-01 20:24:10,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-01 20:24:10,900 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-01 20:24:10,901 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-01 20:24:10,903 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01 20:24:10,903 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-12-01 20:24:10,903 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-01 20:24:10,904 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-12-01T20:24:10,935 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-12-01 20:24:10,937 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-01 20:24:10,938 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01T20:24:11,195 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff 2024-12-01T20:24:11,199 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-12-01T20:24:11,201 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:24:11,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-01T20:24:11,215 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293, deleteOnExit=true 2024-12-01T20:24:11,215 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-01T20:24:11,216 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/test.cache.data in system properties and HBase conf 2024-12-01T20:24:11,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.tmp.dir in system properties and HBase conf 2024-12-01T20:24:11,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir in system properties and HBase conf 2024-12-01T20:24:11,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-01T20:24:11,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-01T20:24:11,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-01T20:24:11,298 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-01T20:24:11,390 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T20:24:11,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-01T20:24:11,395 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-01T20:24:11,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-01T20:24:11,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T20:24:11,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-01T20:24:11,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-01T20:24:11,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T20:24:11,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T20:24:11,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-01T20:24:11,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/nfs.dump.dir in system properties and HBase conf 2024-12-01T20:24:11,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/java.io.tmpdir in system properties and HBase conf 2024-12-01T20:24:11,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T20:24:11,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-01T20:24:11,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-01T20:24:12,494 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-01T20:24:12,592 INFO [Time-limited test {}] log.Log(170): Logging initialized @2563ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-01T20:24:12,705 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:12,778 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T20:24:12,800 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:24:12,801 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:24:12,802 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T20:24:12,822 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:12,825 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f045f3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:24:12,827 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b22dca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T20:24:13,038 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@38e78c7b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/java.io.tmpdir/jetty-localhost-42737-hadoop-hdfs-3_4_1-tests_jar-_-any-13594557171243181517/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T20:24:13,053 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c336369{HTTP/1.1, (http/1.1)}{localhost:42737} 2024-12-01T20:24:13,053 INFO [Time-limited test {}] server.Server(415): Started @3026ms 2024-12-01T20:24:13,639 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:13,646 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T20:24:13,647 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:24:13,647 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:24:13,647 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T20:24:13,648 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:24:13,649 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T20:24:13,744 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@cb44ad4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/java.io.tmpdir/jetty-localhost-35627-hadoop-hdfs-3_4_1-tests_jar-_-any-9896901734118897813/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T20:24:13,745 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:35627} 2024-12-01T20:24:13,745 INFO [Time-limited test {}] server.Server(415): Started @3718ms 2024-12-01T20:24:13,802 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T20:24:14,895 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/data/data1/current/BP-824697111-172.17.0.3-1733084651933/current, will proceed with Du for space computation calculation, 2024-12-01T20:24:14,895 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/data/data2/current/BP-824697111-172.17.0.3-1733084651933/current, will proceed with Du for space computation calculation, 2024-12-01T20:24:14,928 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T20:24:14,975 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xff3d9b03b6bfde87 with lease ID 0x67ec98c1432abab: Processing first storage report for DS-0052a839-d63a-4649-9594-9aab663da30f from datanode DatanodeRegistration(127.0.0.1:36971, datanodeUuid=903de1c7-053c-43f9-b0d3-0c3126b569c3, infoPort=37927, infoSecurePort=0, ipcPort=35827, storageInfo=lv=-57;cid=testClusterID;nsid=566465112;c=1733084651933) 2024-12-01T20:24:14,976 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xff3d9b03b6bfde87 with lease ID 0x67ec98c1432abab: from storage DS-0052a839-d63a-4649-9594-9aab663da30f node DatanodeRegistration(127.0.0.1:36971, datanodeUuid=903de1c7-053c-43f9-b0d3-0c3126b569c3, infoPort=37927, infoSecurePort=0, ipcPort=35827, storageInfo=lv=-57;cid=testClusterID;nsid=566465112;c=1733084651933), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-01T20:24:14,977 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xff3d9b03b6bfde87 with lease ID 0x67ec98c1432abab: Processing first storage report for DS-9d9ca141-37ff-49bc-9a8a-3e0280faaee6 from datanode DatanodeRegistration(127.0.0.1:36971, datanodeUuid=903de1c7-053c-43f9-b0d3-0c3126b569c3, infoPort=37927, infoSecurePort=0, ipcPort=35827, storageInfo=lv=-57;cid=testClusterID;nsid=566465112;c=1733084651933) 2024-12-01T20:24:14,977 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xff3d9b03b6bfde87 with lease ID 0x67ec98c1432abab: from storage DS-9d9ca141-37ff-49bc-9a8a-3e0280faaee6 node DatanodeRegistration(127.0.0.1:36971, datanodeUuid=903de1c7-053c-43f9-b0d3-0c3126b569c3, infoPort=37927, infoSecurePort=0, ipcPort=35827, storageInfo=lv=-57;cid=testClusterID;nsid=566465112;c=1733084651933), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-01T20:24:14,994 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff 2024-12-01T20:24:15,071 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/zookeeper_0, clientPort=57617, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T20:24:15,082 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57617 2024-12-01T20:24:15,098 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:24:15,103 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:24:15,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741825_1001 (size=7) 2024-12-01T20:24:15,742 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e with version=8 2024-12-01T20:24:15,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/hbase-staging 2024-12-01T20:24:15,754 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-12-01T20:24:15,754 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-12-01T20:24:15,754 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-12-01T20:24:15,754 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-12-01T20:24:15,826 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-01T20:24:16,069 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9168bcbe98d9:0 server-side Connection retries=45 2024-12-01T20:24:16,078 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T20:24:16,079 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T20:24:16,083 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T20:24:16,083 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T20:24:16,083 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T20:24:16,208 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T20:24:16,266 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-01T20:24:16,275 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-01T20:24:16,279 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T20:24:16,303 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 160818 (auto-detected) 2024-12-01T20:24:16,304 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-01T20:24:16,322 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39089 2024-12-01T20:24:16,343 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39089 connecting to ZooKeeper ensemble=127.0.0.1:57617 2024-12-01T20:24:16,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390890x0, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T20:24:16,488 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39089-0x101929129790000 connected 2024-12-01T20:24:16,576 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:24:16,578 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:24:16,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-01T20:24:16,607 INFO [Time-limited test {}] log.Log(170): Logging initialized @6580ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-12-01T20:24:16,644 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-12-01T20:24:16,645 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-12-01T20:24:16,645 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-12-01T20:24:16,646 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-12-01T20:24:16,646 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-12-01T20:24:16,648 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-12-01T20:24:16,648 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-12-01T20:24:16,649 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-12-01T20:24:16,650 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-12-01T20:24:16,721 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 41573 2024-12-01T20:24:16,722 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-01T20:24:16,736 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-01T20:24:16,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7c1597dd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:24:16,738 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-01T20:24:16,738 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@65507143{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-12-01T20:24:16,772 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-12-01T20:24:16,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:24:16,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:24:16,780 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T20:24:16,783 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-01T20:24:17,389 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5c685ba8{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-12-01T20:24:17,393 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10f9ba48{HTTP/1.1, (http/1.1)}{0.0.0.0:41573} 2024-12-01T20:24:17,393 INFO [Time-limited test {}] server.Server(415): Started @7365ms 2024-12-01T20:24:17,396 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e, hbase.cluster.distributed=false 2024-12-01T20:24:17,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-12-01T20:24:17,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39089 2024-12-01T20:24:17,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39089 2024-12-01T20:24:17,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39089 2024-12-01T20:24:17,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39089 2024-12-01T20:24:17,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39089 2024-12-01T20:24:17,529 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9168bcbe98d9:0 server-side Connection retries=45 2024-12-01T20:24:17,530 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T20:24:17,530 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T20:24:17,530 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T20:24:17,531 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T20:24:17,531 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T20:24:17,533 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T20:24:17,535 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T20:24:17,536 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38993 2024-12-01T20:24:17,538 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38993 connecting to ZooKeeper ensemble=127.0.0.1:57617 2024-12-01T20:24:17,539 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:24:17,542 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:24:17,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389930x0, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T20:24:17,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:389930x0, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-01T20:24:17,660 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38993-0x101929129790001 connected 2024-12-01T20:24:17,662 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-12-01T20:24:17,662 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-12-01T20:24:17,662 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-12-01T20:24:17,662 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-12-01T20:24:17,662 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-12-01T20:24:17,664 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-12-01T20:24:17,664 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-12-01T20:24:17,664 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-12-01T20:24:17,664 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-12-01T20:24:17,666 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 36627 2024-12-01T20:24:17,666 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-01T20:24:17,668 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-01T20:24:17,668 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3e9615b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:24:17,668 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-01T20:24:17,669 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2df3a35c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-12-01T20:24:17,680 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-12-01T20:24:17,681 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:24:17,681 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:24:17,681 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T20:24:17,682 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-01T20:24:17,685 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@9984275{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-12-01T20:24:17,685 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@445a8045{HTTP/1.1, (http/1.1)}{0.0.0.0:36627} 2024-12-01T20:24:17,685 INFO [Time-limited test {}] server.Server(415): Started @7658ms 2024-12-01T20:24:17,689 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T20:24:17,696 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T20:24:17,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-12-01T20:24:17,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-12-01T20:24:17,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38993 2024-12-01T20:24:17,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38993 2024-12-01T20:24:17,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38993 2024-12-01T20:24:17,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38993 2024-12-01T20:24:17,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38993 2024-12-01T20:24:17,727 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9168bcbe98d9:39089 2024-12-01T20:24:17,730 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-01T20:24:17,732 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@234e862b{HTTP/1.1, (http/1.1)}{0.0.0.0:40641} 2024-12-01T20:24:17,732 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] server.Server(415): Started @7705ms 2024-12-01T20:24:17,733 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-01T20:24:17,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-01T20:24:17,745 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:17,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-12-01T20:24:17,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:17,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:17,775 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/master 2024-12-01T20:24:17,776 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/9168bcbe98d9,39089,1733084655909 from backup master directory 2024-12-01T20:24:17,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:17,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-01T20:24:17,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-01T20:24:17,786 WARN [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T20:24:17,786 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:17,788 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-01T20:24:17,789 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-01T20:24:17,838 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/hbase.id] with ID: 6d28d033-e930-4ea2-a0ce-57e1357eb371 2024-12-01T20:24:17,838 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.tmp/hbase.id 2024-12-01T20:24:17,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741826_1002 (size=42) 2024-12-01T20:24:18,249 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.tmp/hbase.id]:[hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/hbase.id] 2024-12-01T20:24:18,291 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:24:18,295 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T20:24:18,313 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-01T20:24:18,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:18,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:18,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741827_1003 (size=196) 2024-12-01T20:24:18,791 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:24:18,793 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T20:24:18,798 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-01T20:24:18,800 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-01T20:24:18,808 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T20:24:18,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741828_1004 (size=1189) 2024-12-01T20:24:18,854 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store 2024-12-01T20:24:18,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741829_1005 (size=34) 2024-12-01T20:24:18,878 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-01T20:24:18,881 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:18,882 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T20:24:18,882 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:24:18,882 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:24:18,883 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T20:24:18,884 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:24:18,884 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:24:18,885 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733084658882Disabling compacts and flushes for region at 1733084658882Disabling writes for close at 1733084658883 (+1 ms)Writing region close event to WAL at 1733084658884 (+1 ms)Closed at 1733084658884 2024-12-01T20:24:18,887 WARN [master/9168bcbe98d9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/.initializing 2024-12-01T20:24:18,887 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/WALs/9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:18,892 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T20:24:18,896 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9168bcbe98d9,39089,1733084655909.regiongroup-0 2024-12-01T20:24:18,910 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9168bcbe98d9%2C39089%2C1733084655909.9168bcbe98d9%2C39089%2C1733084655909.regiongroup-0, suffix=, logDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/WALs/9168bcbe98d9,39089,1733084655909, archiveDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/oldWALs, maxLogs=10 2024-12-01T20:24:18,932 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/WALs/9168bcbe98d9,39089,1733084655909/9168bcbe98d9%2C39089%2C1733084655909.9168bcbe98d9%2C39089%2C1733084655909.regiongroup-0.1733084658914, exclude list is [], retry=0 2024-12-01T20:24:18,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:24:18,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-01T20:24:18,982 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/WALs/9168bcbe98d9,39089,1733084655909/9168bcbe98d9%2C39089%2C1733084655909.9168bcbe98d9%2C39089%2C1733084655909.regiongroup-0.1733084658914 2024-12-01T20:24:18,983 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:24:18,984 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:18,984 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:18,987 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:18,988 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,055 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T20:24:19,059 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:19,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:19,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T20:24:19,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:19,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:19,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T20:24:19,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:19,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:19,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,076 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T20:24:19,076 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:19,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:19,078 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,081 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,083 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,089 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,090 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,093 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T20:24:19,098 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:24:19,102 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:24:19,104 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9767091520, jitterRate=-0.09036871790885925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T20:24:19,110 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733084659003Initializing all the Stores at 1733084659006 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733084659006Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084659008 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084659008Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084659008Cleaning up temporary data from old regions at 1733084659090 (+82 ms)Region opened successfully at 1733084659110 (+20 ms) 2024-12-01T20:24:19,123 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T20:24:19,159 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402d9eb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=9168bcbe98d9/172.17.0.3:0 2024-12-01T20:24:19,189 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T20:24:19,199 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T20:24:19,199 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T20:24:19,202 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T20:24:19,203 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-01T20:24:19,208 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-01T20:24:19,209 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T20:24:19,232 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T20:24:19,257 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-12-01T20:24:19,321 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-12-01T20:24:19,324 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T20:24:19,326 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-12-01T20:24:19,332 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-12-01T20:24:19,334 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T20:24:19,338 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-12-01T20:24:19,351 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-12-01T20:24:19,352 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-12-01T20:24:19,363 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-12-01T20:24:19,379 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T20:24:19,445 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-12-01T20:24:19,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-12-01T20:24:19,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-12-01T20:24:19,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:19,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:19,482 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9168bcbe98d9,39089,1733084655909, sessionid=0x101929129790000, setting cluster-up flag (Was=false) 2024-12-01T20:24:19,489 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-12-01T20:24:19,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:19,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:19,542 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-12-01T20:24:19,544 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:19,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:19,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:19,595 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-12-01T20:24:19,597 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:19,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:19,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:19,648 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-12-01T20:24:19,650 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:19,656 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T20:24:19,712 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(746): ClusterId : 6d28d033-e930-4ea2-a0ce-57e1357eb371 2024-12-01T20:24:19,714 INFO [RS:0;9168bcbe98d9:38993 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-12-01T20:24:19,716 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T20:24:19,726 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T20:24:19,732 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T20:24:19,732 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-12-01T20:24:19,736 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T20:24:19,742 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T20:24:19,744 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-12-01T20:24:19,744 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T20:24:19,747 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9168bcbe98d9,39089,1733084655909 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T20:24:19,754 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T20:24:19,755 DEBUG [RS:0;9168bcbe98d9:38993 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4266b908, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=9168bcbe98d9/172.17.0.3:0 2024-12-01T20:24:19,755 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9168bcbe98d9:0, corePoolSize=5, maxPoolSize=5 2024-12-01T20:24:19,756 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9168bcbe98d9:0, corePoolSize=5, maxPoolSize=5 2024-12-01T20:24:19,756 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9168bcbe98d9:0, corePoolSize=5, maxPoolSize=5 2024-12-01T20:24:19,756 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9168bcbe98d9:0, corePoolSize=5, maxPoolSize=5 2024-12-01T20:24:19,756 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9168bcbe98d9:0, corePoolSize=10, maxPoolSize=10 2024-12-01T20:24:19,756 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:19,757 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9168bcbe98d9:0, corePoolSize=2, maxPoolSize=2 2024-12-01T20:24:19,757 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:19,761 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733084689761 2024-12-01T20:24:19,763 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T20:24:19,764 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T20:24:19,765 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T20:24:19,766 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T20:24:19,768 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9168bcbe98d9:38993 2024-12-01T20:24:19,768 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T20:24:19,769 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-12-01T20:24:19,769 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T20:24:19,769 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T20:24:19,769 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T20:24:19,770 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T20:24:19,771 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T20:24:19,771 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T20:24:19,772 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:19,772 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T20:24:19,771 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:19,773 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(2659): reportForDuty to master=9168bcbe98d9,39089,1733084655909 with port=38993, startcode=1733084657496 2024-12-01T20:24:19,775 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T20:24:19,776 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T20:24:19,776 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T20:24:19,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741831_1007 (size=1321) 2024-12-01T20:24:19,787 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f865ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:19,788 DEBUG [RS:0;9168bcbe98d9:38993 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T20:24:19,788 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:24:19,791 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:24:19,810 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:24:19,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:24:19,821 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56677, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T20:24:19,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:24:19,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ed492e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:19,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:24:19,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:24:19,826 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-01T20:24:19,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:19,828 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:24:19,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4de8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:19,833 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-12-01T20:24:19,835 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T20:24:19,835 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T20:24:19,837 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9168bcbe98d9:0:becomeActiveMaster-HFileCleaner.large.0-1733084659837,5,FailOnTimeoutGroup] 2024-12-01T20:24:19,838 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9168bcbe98d9:0:becomeActiveMaster-HFileCleaner.small.0-1733084659837,5,FailOnTimeoutGroup] 2024-12-01T20:24:19,838 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:19,838 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T20:24:19,839 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:19,839 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:19,847 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-01T20:24:19,847 WARN [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-01T20:24:19,948 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(2659): reportForDuty to master=9168bcbe98d9,39089,1733084655909 with port=38993, startcode=1733084657496 2024-12-01T20:24:19,950 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39089 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:19,952 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39089 {}] master.ServerManager(517): Registering regionserver=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:19,959 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:19,959 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44065 2024-12-01T20:24:19,959 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=41573 2024-12-01T20:24:20,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-12-01T20:24:20,001 DEBUG [RS:0;9168bcbe98d9:38993 {}] zookeeper.ZKUtil(111): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rs/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:20,001 WARN [RS:0;9168bcbe98d9:38993 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T20:24:20,001 INFO [RS:0;9168bcbe98d9:38993 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-01T20:24:20,001 INFO [RS:0;9168bcbe98d9:38993 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-01T20:24:20,002 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:20,003 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9168bcbe98d9,38993,1733084657496] 2024-12-01T20:24:20,024 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T20:24:20,035 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T20:24:20,040 INFO [RS:0;9168bcbe98d9:38993 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T20:24:20,040 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,041 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T20:24:20,046 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T20:24:20,048 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,048 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,048 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,048 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,048 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,049 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,049 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9168bcbe98d9:0, corePoolSize=2, maxPoolSize=2 2024-12-01T20:24:20,049 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,049 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,049 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,049 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,049 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,049 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9168bcbe98d9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:24:20,050 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0, corePoolSize=3, maxPoolSize=3 2024-12-01T20:24:20,050 DEBUG [RS:0;9168bcbe98d9:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9168bcbe98d9:0, corePoolSize=3, maxPoolSize=3 2024-12-01T20:24:20,050 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,051 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,051 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,051 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,051 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,051 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=9168bcbe98d9,38993,1733084657496-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T20:24:20,067 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T20:24:20,069 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=9168bcbe98d9,38993,1733084657496-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,069 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,069 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.Replication(171): 9168bcbe98d9,38993,1733084657496 started 2024-12-01T20:24:20,084 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:20,085 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1482): Serving as 9168bcbe98d9,38993,1733084657496, RpcServer on 9168bcbe98d9/172.17.0.3:38993, sessionid=0x101929129790001 2024-12-01T20:24:20,085 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T20:24:20,086 DEBUG [RS:0;9168bcbe98d9:38993 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:20,086 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9168bcbe98d9,38993,1733084657496' 2024-12-01T20:24:20,086 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-12-01T20:24:20,087 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-12-01T20:24:20,088 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T20:24:20,088 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-12-01T20:24:20,088 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9168bcbe98d9,38993,1733084657496' 2024-12-01T20:24:20,088 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:24:20,089 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:24:20,089 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-12-01T20:24:20,089 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-12-01T20:24:20,089 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T20:24:20,089 DEBUG [RS:0;9168bcbe98d9:38993 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:20,090 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9168bcbe98d9,38993,1733084657496' 2024-12-01T20:24:20,090 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-12-01T20:24:20,090 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-12-01T20:24:20,091 DEBUG [RS:0;9168bcbe98d9:38993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T20:24:20,091 INFO [RS:0;9168bcbe98d9:38993 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T20:24:20,091 INFO [RS:0;9168bcbe98d9:38993 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T20:24:20,187 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T20:24:20,188 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:20,196 INFO [RS:0;9168bcbe98d9:38993 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T20:24:20,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741832_1008 (size=32) 2024-12-01T20:24:20,196 INFO [RS:0;9168bcbe98d9:38993 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9168bcbe98d9,38993,1733084657496.regiongroup-0 2024-12-01T20:24:20,199 INFO [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0, suffix=, logDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496, archiveDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs, maxLogs=32 2024-12-01T20:24:20,212 DEBUG [RS:0;9168bcbe98d9:38993 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084660201, exclude list is [], retry=0 2024-12-01T20:24:20,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:24:20,219 INFO [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084660201 2024-12-01T20:24:20,220 DEBUG [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:24:20,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:20,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T20:24:20,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T20:24:20,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:20,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:20,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T20:24:20,607 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T20:24:20,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:20,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:20,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T20:24:20,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T20:24:20,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:20,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:20,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T20:24:20,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T20:24:20,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:20,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:20,615 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T20:24:20,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740 2024-12-01T20:24:20,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740 2024-12-01T20:24:20,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T20:24:20,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T20:24:20,621 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T20:24:20,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T20:24:20,627 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:24:20,627 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10745550400, jitterRate=7.57366418838501E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T20:24:20,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733084660599Initializing all the Stores at 1733084660601 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733084660601Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733084660601Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084660601Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733084660601Cleaning up temporary data from old regions at 1733084660620 (+19 ms)Region opened successfully at 1733084660629 (+9 ms) 2024-12-01T20:24:20,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T20:24:20,629 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T20:24:20,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T20:24:20,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T20:24:20,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T20:24:20,631 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T20:24:20,631 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733084660629Disabling compacts and flushes for region at 1733084660629Disabling writes for close at 1733084660629Writing region close event to WAL at 1733084660630 (+1 ms)Closed at 1733084660631 (+1 ms) 2024-12-01T20:24:20,634 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T20:24:20,634 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T20:24:20,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T20:24:20,647 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T20:24:20,650 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T20:24:20,803 DEBUG [9168bcbe98d9:39089 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-01T20:24:20,815 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:20,821 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9168bcbe98d9,38993,1733084657496, state=OPENING 2024-12-01T20:24:20,869 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T20:24:20,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:20,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:24:20,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-01T20:24:20,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-01T20:24:20,881 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T20:24:20,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:21,055 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T20:24:21,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38311, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T20:24:21,066 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T20:24:21,067 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-01T20:24:21,067 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-01T20:24:21,067 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T20:24:21,068 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T20:24:21,071 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9168bcbe98d9%2C38993%2C1733084657496.meta, suffix=.meta, logDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496, archiveDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs, maxLogs=32 2024-12-01T20:24:21,085 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084661072.meta, exclude list is [], retry=0 2024-12-01T20:24:21,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:24:21,091 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084661072.meta 2024-12-01T20:24:21,092 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:24:21,092 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:21,096 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:24:21,097 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T20:24:21,099 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T20:24:21,099 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T20:24:21,103 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T20:24:21,103 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:21,104 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T20:24:21,104 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T20:24:21,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T20:24:21,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T20:24:21,109 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:21,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:21,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T20:24:21,113 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T20:24:21,113 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:21,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:21,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T20:24:21,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T20:24:21,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:21,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:21,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T20:24:21,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T20:24:21,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:21,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:24:21,121 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T20:24:21,122 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740 2024-12-01T20:24:21,125 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740 2024-12-01T20:24:21,127 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T20:24:21,127 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T20:24:21,128 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T20:24:21,131 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T20:24:21,132 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10342356480, jitterRate=-0.03679299354553223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T20:24:21,132 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T20:24:21,133 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733084661104Writing region info on filesystem at 1733084661104Initializing all the Stores at 1733084661106 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733084661106Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733084661107 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084661107Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733084661107Cleaning up temporary data from old regions at 1733084661127 (+20 ms)Running coprocessor post-open hooks at 1733084661132 (+5 ms)Region opened successfully at 1733084661133 (+1 ms) 2024-12-01T20:24:21,140 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733084661047 2024-12-01T20:24:21,151 DEBUG [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T20:24:21,152 INFO [RS_OPEN_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T20:24:21,153 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:21,157 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9168bcbe98d9,38993,1733084657496, state=OPEN 2024-12-01T20:24:21,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-12-01T20:24:21,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-12-01T20:24:21,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-01T20:24:21,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-01T20:24:21,164 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:21,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T20:24:21,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9168bcbe98d9,38993,1733084657496 in 281 msec 2024-12-01T20:24:21,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T20:24:21,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 533 msec 2024-12-01T20:24:21,179 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T20:24:21,179 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T20:24:21,198 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:21,199 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:21,219 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:21,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60583, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:24:21,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5580 sec 2024-12-01T20:24:21,244 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733084661244, completionTime=-1 2024-12-01T20:24:21,246 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-01T20:24:21,247 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T20:24:21,275 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-01T20:24:21,275 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733084721275 2024-12-01T20:24:21,275 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733084781275 2024-12-01T20:24:21,275 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-01T20:24:21,287 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9168bcbe98d9,39089,1733084655909-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:21,288 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9168bcbe98d9,39089,1733084655909-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:21,288 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9168bcbe98d9,39089,1733084655909-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:21,290 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9168bcbe98d9:39089, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:21,290 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:21,291 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:21,297 DEBUG [master/9168bcbe98d9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T20:24:21,320 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.534sec 2024-12-01T20:24:21,321 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T20:24:21,323 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T20:24:21,324 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T20:24:21,324 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T20:24:21,324 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T20:24:21,325 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9168bcbe98d9,39089,1733084655909-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T20:24:21,325 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9168bcbe98d9,39089,1733084655909-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T20:24:21,342 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T20:24:21,343 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T20:24:21,344 INFO [master/9168bcbe98d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9168bcbe98d9,39089,1733084655909-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:24:21,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cc25be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:21,417 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-01T20:24:21,417 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-01T20:24:21,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:24:21,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:24:21,427 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:24:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:24:21,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:24:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d1868b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:24:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:24:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:21,430 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:24:21,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195abe33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:21,432 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:21,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:24:21,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:21,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-12-01T20:24:21,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/test.cache.data in system properties and HBase conf 2024-12-01T20:24:21,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.tmp.dir in system properties and HBase conf 2024-12-01T20:24:21,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir in system properties and HBase conf 2024-12-01T20:24:21,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/nfs.dump.dir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/java.io.tmpdir in system properties and HBase conf 2024-12-01T20:24:21,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T20:24:21,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-01T20:24:21,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-01T20:24:21,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741835_1011 (size=592039) 2024-12-01T20:24:21,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741836_1012 (size=1663647) 2024-12-01T20:24:23,256 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:23,329 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:23,524 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T20:24:23,525 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-01T20:24:23,526 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T20:24:23,526 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:24:23,526 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:24:23,526 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-01T20:24:23,527 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:24:23,527 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:24:23,527 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T20:24:23,527 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68f08892{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:24:23,528 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d8c4596{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-01T20:24:23,529 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:23,541 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ccdffea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:24:23,542 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31795375{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-01T20:24:23,658 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2024-12-01T20:24:23,658 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2024-12-01T20:24:23,659 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-01T20:24:23,660 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-01T20:24:23,703 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:23,815 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:24,041 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:24,069 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4aaa7aca{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/java.io.tmpdir/jetty-localhost-36685-hadoop-yarn-common-3_4_1_jar-_-any-4514593873616084495/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-12-01T20:24:24,069 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12da9210{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/java.io.tmpdir/jetty-localhost-33433-hadoop-yarn-common-3_4_1_jar-_-any-10986175341970501838/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-12-01T20:24:24,069 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@4c920e7f{HTTP/1.1, (http/1.1)}{localhost:36685} 2024-12-01T20:24:24,070 INFO [Thread-251 {}] server.Server(415): Started @14042ms 2024-12-01T20:24:24,070 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@567287a2{HTTP/1.1, (http/1.1)}{localhost:33433} 2024-12-01T20:24:24,071 INFO [Time-limited test {}] server.Server(415): Started @14043ms 2024-12-01T20:24:24,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741837_1013 (size=5) 2024-12-01T20:24:25,232 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-12-01T20:24:25,235 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:25,256 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-01T20:24:25,257 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T20:24:25,259 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:24:25,259 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:24:25,259 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T20:24:25,260 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:25,260 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29611ace{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:24:25,261 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435b70f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-01T20:24:25,328 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-12-01T20:24:25,328 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-01T20:24:25,328 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-12-01T20:24:25,329 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-01T20:24:25,337 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:25,343 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:25,430 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:25,445 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@482c666e{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/java.io.tmpdir/jetty-localhost-45431-hadoop-yarn-common-3_4_1_jar-_-any-11007531675366714796/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-01T20:24:25,446 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e7d9d04{HTTP/1.1, (http/1.1)}{localhost:45431} 2024-12-01T20:24:25,446 INFO [Time-limited test {}] server.Server(415): Started @15419ms 2024-12-01T20:24:25,586 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-12-01T20:24:25,589 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:25,603 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-01T20:24:25,604 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T20:24:25,608 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:24:25,608 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:24:25,608 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-01T20:24:25,609 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:24:25,611 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f6ea73d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:24:25,611 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18552df3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-01T20:24:25,656 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-12-01T20:24:25,656 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-01T20:24:25,656 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-12-01T20:24:25,656 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-01T20:24:25,664 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:25,668 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:25,739 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-01T20:24:25,744 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5bf7740{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/java.io.tmpdir/jetty-localhost-46167-hadoop-yarn-common-3_4_1_jar-_-any-12107013196671816087/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-01T20:24:25,745 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f062696{HTTP/1.1, (http/1.1)}{localhost:46167} 2024-12-01T20:24:25,745 INFO [Time-limited test {}] server.Server(415): Started @15717ms 2024-12-01T20:24:25,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-12-01T20:24:25,768 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:44065/backupUT 2024-12-01T20:24:25,772 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T20:24:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@13891414 2024-12-01T20:24:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T20:24:25,784 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T20:24:25,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns1'} 2024-12-01T20:24:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-12-01T20:24:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T20:24:25,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 21 msec 2024-12-01T20:24:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T20:24:25,918 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-12-01T20:24:25,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns2'} 2024-12-01T20:24:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-12-01T20:24:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-12-01T20:24:25,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 12 msec 2024-12-01T20:24:26,021 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T20:24:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-12-01T20:24:26,028 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-12-01T20:24:26,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns3'} 2024-12-01T20:24:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-12-01T20:24:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-12-01T20:24:26,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 12 msec 2024-12-01T20:24:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-12-01T20:24:26,138 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-12-01T20:24:26,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'ns4'} 2024-12-01T20:24:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-12-01T20:24:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-01T20:24:26,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 13 msec 2024-12-01T20:24:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-01T20:24:26,258 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-12-01T20:24:26,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test-1733084665768', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:24:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1733084665768 2024-12-01T20:24:26,275 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1733084665768 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:24:26,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test-1733084665768" procId is: 8 2024-12-01T20:24:26,277 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:26,279 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1733084665768 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:24:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-01T20:24:26,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741838_1014 (size=367) 2024-12-01T20:24:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-01T20:24:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-01T20:24:26,614 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-01T20:24:26,677 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-01T20:24:26,677 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-01T20:24:26,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:24:26,680 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-12-01T20:24:26,680 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T20:24:26,680 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-01T20:24:26,680 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-01T20:24:26,680 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-01T20:24:26,693 INFO [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00ff3d9048bd8163f6ce55a5b39c9f69, NAME => 'test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1733084665768', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:26,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741839_1015 (size=53) 2024-12-01T20:24:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-01T20:24:27,102 DEBUG [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(898): Instantiated test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:27,103 DEBUG [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(1722): Closing 00ff3d9048bd8163f6ce55a5b39c9f69, disabling compactions & flushes 2024-12-01T20:24:27,103 INFO [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(1755): Closing region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:27,103 DEBUG [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:27,103 DEBUG [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. after waiting 0 ms 2024-12-01T20:24:27,103 DEBUG [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:27,103 INFO [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(1973): Closed test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:27,103 DEBUG [RegionOpenAndInit-test-1733084665768-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00ff3d9048bd8163f6ce55a5b39c9f69: Waiting for close lock at 1733084667102Disabling compacts and flushes for region at 1733084667103 (+1 ms)Disabling writes for close at 1733084667103Writing region close event to WAL at 1733084667103Closed at 1733084667103 2024-12-01T20:24:27,105 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1733084665768 execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:24:27,110 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733084667105"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733084667105"}]},"ts":"1733084667105"} 2024-12-01T20:24:27,114 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:24:27,116 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1733084665768 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:24:27,119 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1733084665768","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084667116"}]},"ts":"1733084667116"} 2024-12-01T20:24:27,123 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1733084665768, state=ENABLING in hbase:meta 2024-12-01T20:24:27,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, ASSIGN}] 2024-12-01T20:24:27,127 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, ASSIGN 2024-12-01T20:24:27,129 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, ASSIGN; state=OFFLINE, location=9168bcbe98d9,38993,1733084657496; forceNewPlan=false, retain=false 2024-12-01T20:24:27,280 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:27,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, ASSIGN because future has completed 2024-12-01T20:24:27,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-01T20:24:27,464 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:27,465 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T20:24:27,465 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor 9168bcbe98d9,38993,1733084657496.regiongroup-1 2024-12-01T20:24:27,471 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1, suffix=, logDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496, archiveDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs, maxLogs=32 2024-12-01T20:24:27,484 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084667472, exclude list is [], retry=0 2024-12-01T20:24:27,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:24:27,491 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084667472 2024-12-01T20:24:27,491 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:24:27,491 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 00ff3d9048bd8163f6ce55a5b39c9f69, NAME => 'test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:27,492 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:24:27,492 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733084665768 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,492 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:27,492 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,492 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,495 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,498 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ff3d9048bd8163f6ce55a5b39c9f69 columnFamilyName f 2024-12-01T20:24:27,498 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:27,499 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(327): Store=00ff3d9048bd8163f6ce55a5b39c9f69/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:27,500 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,501 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,502 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,502 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,502 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,505 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,509 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:24:27,510 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 00ff3d9048bd8163f6ce55a5b39c9f69; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9774298880, jitterRate=-0.08969748020172119}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:24:27,510 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:27,511 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 00ff3d9048bd8163f6ce55a5b39c9f69: Running coprocessor pre-open hook at 1733084667492Writing region info on filesystem at 1733084667492Initializing all the Stores at 1733084667494 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084667494Cleaning up temporary data from old regions at 1733084667502 (+8 ms)Running coprocessor post-open hooks at 1733084667510 (+8 ms)Region opened successfully at 1733084667511 (+1 ms) 2024-12-01T20:24:27,513 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., pid=10, masterSystemTime=1733084667437 2024-12-01T20:24:27,517 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:27,517 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:27,518 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=OPEN, openSeqNum=2, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:27,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:24:27,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-01T20:24:27,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 in 242 msec 2024-12-01T20:24:27,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-01T20:24:27,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, ASSIGN in 406 msec 2024-12-01T20:24:27,537 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1733084665768 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:24:27,537 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1733084665768","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084667537"}]},"ts":"1733084667537"} 2024-12-01T20:24:27,541 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1733084665768, state=ENABLED in hbase:meta 2024-12-01T20:24:27,542 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1733084665768 execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:24:27,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1733084665768 in 1.2750 sec 2024-12-01T20:24:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-01T20:24:28,428 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1733084665768 completed 2024-12-01T20:24:28,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bfa997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:28,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:24:28,430 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:24:28,431 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:24:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:24:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@191c23ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:24:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:28,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:24:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bd77c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:28,440 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:28,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:28,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:24:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733084665768', row='row0', locateType=CURRENT is [region=test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., hostname=9168bcbe98d9,38993,1733084657496, seqNum=2] 2024-12-01T20:24:28,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(8528): writing data to region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. with WAL disabled. Data may be lost in the event of a crash. 2024-12-01T20:24:28,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns2:test-17330846657681', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:24:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17330846657681 2024-12-01T20:24:28,638 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17330846657681 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:24:28,639 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:28,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns2" qualifier: "test-17330846657681" procId is: 11 2024-12-01T20:24:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-01T20:24:28,640 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17330846657681 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:24:28,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741841_1017 (size=364) 2024-12-01T20:24:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-01T20:24:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-01T20:24:29,052 INFO [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a3c8df032a6e62b8e874b176ca842b1a, NAME => 'ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17330846657681', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:29,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741842_1018 (size=50) 2024-12-01T20:24:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-01T20:24:29,488 DEBUG [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:29,488 DEBUG [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(1722): Closing a3c8df032a6e62b8e874b176ca842b1a, disabling compactions & flushes 2024-12-01T20:24:29,488 INFO [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:29,488 DEBUG [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:29,488 DEBUG [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. after waiting 0 ms 2024-12-01T20:24:29,488 DEBUG [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:29,488 INFO [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:29,489 DEBUG [RegionOpenAndInit-ns2:test-17330846657681-pool-0 {}] regionserver.HRegion(1676): Region close journal for a3c8df032a6e62b8e874b176ca842b1a: Waiting for close lock at 1733084669488Disabling compacts and flushes for region at 1733084669488Disabling writes for close at 1733084669488Writing region close event to WAL at 1733084669488Closed at 1733084669488 2024-12-01T20:24:29,490 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17330846657681 execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:24:29,491 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733084669490"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733084669490"}]},"ts":"1733084669490"} 2024-12-01T20:24:29,494 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:24:29,495 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17330846657681 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:24:29,496 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17330846657681","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084669495"}]},"ts":"1733084669495"} 2024-12-01T20:24:29,498 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17330846657681, state=ENABLING in hbase:meta 2024-12-01T20:24:29,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17330846657681, region=a3c8df032a6e62b8e874b176ca842b1a, ASSIGN}] 2024-12-01T20:24:29,500 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17330846657681, region=a3c8df032a6e62b8e874b176ca842b1a, ASSIGN 2024-12-01T20:24:29,502 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17330846657681, region=a3c8df032a6e62b8e874b176ca842b1a, ASSIGN; state=OFFLINE, location=9168bcbe98d9,38993,1733084657496; forceNewPlan=false, retain=false 2024-12-01T20:24:29,653 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=a3c8df032a6e62b8e874b176ca842b1a, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:29,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17330846657681, region=a3c8df032a6e62b8e874b176ca842b1a, ASSIGN because future has completed 2024-12-01T20:24:29,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3c8df032a6e62b8e874b176ca842b1a, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-01T20:24:29,816 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:29,816 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => a3c8df032a6e62b8e874b176ca842b1a, NAME => 'ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:29,816 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:24:29,816 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17330846657681 a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,817 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:29,817 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,817 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,819 INFO [StoreOpener-a3c8df032a6e62b8e874b176ca842b1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,821 INFO [StoreOpener-a3c8df032a6e62b8e874b176ca842b1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3c8df032a6e62b8e874b176ca842b1a columnFamilyName f 2024-12-01T20:24:29,821 DEBUG [StoreOpener-a3c8df032a6e62b8e874b176ca842b1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:29,822 INFO [StoreOpener-a3c8df032a6e62b8e874b176ca842b1a-1 {}] regionserver.HStore(327): Store=a3c8df032a6e62b8e874b176ca842b1a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:29,822 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,824 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,824 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,825 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,825 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,828 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,831 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:24:29,832 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened a3c8df032a6e62b8e874b176ca842b1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12037919360, jitterRate=0.12111860513687134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:24:29,832 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:29,833 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for a3c8df032a6e62b8e874b176ca842b1a: Running coprocessor pre-open hook at 1733084669817Writing region info on filesystem at 1733084669817Initializing all the Stores at 1733084669818 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084669818Cleaning up temporary data from old regions at 1733084669825 (+7 ms)Running coprocessor post-open hooks at 1733084669833 (+8 ms)Region opened successfully at 1733084669833 2024-12-01T20:24:29,835 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a., pid=13, masterSystemTime=1733084669810 2024-12-01T20:24:29,838 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:29,838 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:29,839 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=a3c8df032a6e62b8e874b176ca842b1a, regionState=OPEN, openSeqNum=2, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:29,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3c8df032a6e62b8e874b176ca842b1a, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:24:29,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-12-01T20:24:29,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure a3c8df032a6e62b8e874b176ca842b1a, server=9168bcbe98d9,38993,1733084657496 in 187 msec 2024-12-01T20:24:29,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-01T20:24:29,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17330846657681, region=a3c8df032a6e62b8e874b176ca842b1a, ASSIGN in 349 msec 2024-12-01T20:24:29,854 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17330846657681 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:24:29,854 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17330846657681","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084669854"}]},"ts":"1733084669854"} 2024-12-01T20:24:29,857 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17330846657681, state=ENABLED in hbase:meta 2024-12-01T20:24:29,859 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17330846657681 execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:24:29,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17330846657681 in 1.2240 sec 2024-12-01T20:24:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-01T20:24:30,787 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17330846657681 completed 2024-12-01T20:24:30,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17330846657681', row='row0', locateType=CURRENT is [region=ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a., hostname=9168bcbe98d9,38993,1733084657496, seqNum=2] 2024-12-01T20:24:30,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] regionserver.HRegion(8528): writing data to region ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. with WAL disabled. Data may be lost in the event of a crash. 2024-12-01T20:24:30,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns3:test-17330846657682', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:24:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17330846657682 2024-12-01T20:24:30,914 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17330846657682 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:24:30,914 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:30,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns3" qualifier: "test-17330846657682" procId is: 14 2024-12-01T20:24:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-01T20:24:30,915 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17330846657682 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:24:30,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741843_1019 (size=390) 2024-12-01T20:24:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-01T20:24:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-01T20:24:31,326 INFO [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb2648ec811d6e10defe644f33b327cd, NAME => 'ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17330846657682', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:31,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741844_1020 (size=50) 2024-12-01T20:24:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-01T20:24:31,735 DEBUG [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:31,735 DEBUG [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(1722): Closing fb2648ec811d6e10defe644f33b327cd, disabling compactions & flushes 2024-12-01T20:24:31,735 INFO [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:24:31,736 DEBUG [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:24:31,736 DEBUG [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. after waiting 0 ms 2024-12-01T20:24:31,736 DEBUG [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:24:31,736 INFO [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:24:31,736 DEBUG [RegionOpenAndInit-ns3:test-17330846657682-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb2648ec811d6e10defe644f33b327cd: Waiting for close lock at 1733084671735Disabling compacts and flushes for region at 1733084671735Disabling writes for close at 1733084671736 (+1 ms)Writing region close event to WAL at 1733084671736Closed at 1733084671736 2024-12-01T20:24:31,738 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17330846657682 execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:24:31,739 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733084671738"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733084671738"}]},"ts":"1733084671738"} 2024-12-01T20:24:31,742 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:24:31,744 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17330846657682 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:24:31,745 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17330846657682","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084671744"}]},"ts":"1733084671744"} 2024-12-01T20:24:31,747 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17330846657682, state=ENABLING in hbase:meta 2024-12-01T20:24:31,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17330846657682, region=fb2648ec811d6e10defe644f33b327cd, ASSIGN}] 2024-12-01T20:24:31,749 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17330846657682, region=fb2648ec811d6e10defe644f33b327cd, ASSIGN 2024-12-01T20:24:31,751 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17330846657682, region=fb2648ec811d6e10defe644f33b327cd, ASSIGN; state=OFFLINE, location=9168bcbe98d9,38993,1733084657496; forceNewPlan=false, retain=false 2024-12-01T20:24:31,901 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=fb2648ec811d6e10defe644f33b327cd, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:31,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17330846657682, region=fb2648ec811d6e10defe644f33b327cd, ASSIGN because future has completed 2024-12-01T20:24:31,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb2648ec811d6e10defe644f33b327cd, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:32,023 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17330846657681' 2024-12-01T20:24:32,024 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1733084665768' 2024-12-01T20:24:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-01T20:24:32,065 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:24:32,065 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => fb2648ec811d6e10defe644f33b327cd, NAME => 'ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:32,065 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:24:32,066 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17330846657682 fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,066 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:32,066 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,066 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,069 INFO [StoreOpener-fb2648ec811d6e10defe644f33b327cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,072 INFO [StoreOpener-fb2648ec811d6e10defe644f33b327cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb2648ec811d6e10defe644f33b327cd columnFamilyName f 2024-12-01T20:24:32,072 DEBUG [StoreOpener-fb2648ec811d6e10defe644f33b327cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:32,073 INFO [StoreOpener-fb2648ec811d6e10defe644f33b327cd-1 {}] regionserver.HStore(327): Store=fb2648ec811d6e10defe644f33b327cd/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:32,073 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,074 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns3/test-17330846657682/fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,075 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns3/test-17330846657682/fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,076 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,076 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,079 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,083 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns3/test-17330846657682/fb2648ec811d6e10defe644f33b327cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:24:32,084 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened fb2648ec811d6e10defe644f33b327cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10184559040, jitterRate=-0.051489025354385376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:24:32,084 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:24:32,084 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for fb2648ec811d6e10defe644f33b327cd: Running coprocessor pre-open hook at 1733084672066Writing region info on filesystem at 1733084672066Initializing all the Stores at 1733084672068 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084672069 (+1 ms)Cleaning up temporary data from old regions at 1733084672076 (+7 ms)Running coprocessor post-open hooks at 1733084672084 (+8 ms)Region opened successfully at 1733084672084 2024-12-01T20:24:32,086 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd., pid=16, masterSystemTime=1733084672059 2024-12-01T20:24:32,089 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:24:32,089 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:24:32,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=fb2648ec811d6e10defe644f33b327cd, regionState=OPEN, openSeqNum=2, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:32,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb2648ec811d6e10defe644f33b327cd, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:24:32,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-01T20:24:32,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure fb2648ec811d6e10defe644f33b327cd, server=9168bcbe98d9,38993,1733084657496 in 188 msec 2024-12-01T20:24:32,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-01T20:24:32,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17330846657682, region=fb2648ec811d6e10defe644f33b327cd, ASSIGN in 349 msec 2024-12-01T20:24:32,102 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17330846657682 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:24:32,102 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17330846657682","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084672102"}]},"ts":"1733084672102"} 2024-12-01T20:24:32,105 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17330846657682, state=ENABLED in hbase:meta 2024-12-01T20:24:32,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17330846657682 execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:24:32,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17330846657682 in 1.1960 sec 2024-12-01T20:24:32,995 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-01T20:24:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-01T20:24:33,058 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17330846657682 completed 2024-12-01T20:24:33,058 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17330846657682 get assigned. Timeout = 60000ms 2024-12-01T20:24:33,059 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:24:33,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17330846657682 assigned to meta. Checking AM states. 2024-12-01T20:24:33,066 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:24:33,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17330846657682 assigned. 2024-12-01T20:24:33,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'ns4:test-17330846657683', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:24:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17330846657683 2024-12-01T20:24:33,072 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17330846657683 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:24:33,072 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:33,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "ns4" qualifier: "test-17330846657683" procId is: 17 2024-12-01T20:24:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-01T20:24:33,073 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17330846657683 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:24:33,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741845_1021 (size=390) 2024-12-01T20:24:33,087 INFO [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ea117edd9c2b760fcdfda100b3c4ed5c, NAME => 'ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17330846657683', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:33,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741846_1022 (size=50) 2024-12-01T20:24:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-01T20:24:33,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-01T20:24:33,497 DEBUG [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:33,497 DEBUG [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(1722): Closing ea117edd9c2b760fcdfda100b3c4ed5c, disabling compactions & flushes 2024-12-01T20:24:33,497 INFO [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:24:33,497 DEBUG [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:24:33,497 DEBUG [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. after waiting 0 ms 2024-12-01T20:24:33,498 DEBUG [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:24:33,498 INFO [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:24:33,498 DEBUG [RegionOpenAndInit-ns4:test-17330846657683-pool-0 {}] regionserver.HRegion(1676): Region close journal for ea117edd9c2b760fcdfda100b3c4ed5c: Waiting for close lock at 1733084673497Disabling compacts and flushes for region at 1733084673497Disabling writes for close at 1733084673497Writing region close event to WAL at 1733084673498 (+1 ms)Closed at 1733084673498 2024-12-01T20:24:33,499 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17330846657683 execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:24:33,500 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733084673499"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733084673499"}]},"ts":"1733084673499"} 2024-12-01T20:24:33,507 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:24:33,508 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17330846657683 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:24:33,509 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17330846657683","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084673508"}]},"ts":"1733084673508"} 2024-12-01T20:24:33,511 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17330846657683, state=ENABLING in hbase:meta 2024-12-01T20:24:33,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17330846657683, region=ea117edd9c2b760fcdfda100b3c4ed5c, ASSIGN}] 2024-12-01T20:24:33,513 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17330846657683, region=ea117edd9c2b760fcdfda100b3c4ed5c, ASSIGN 2024-12-01T20:24:33,514 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17330846657683, region=ea117edd9c2b760fcdfda100b3c4ed5c, ASSIGN; state=OFFLINE, location=9168bcbe98d9,38993,1733084657496; forceNewPlan=false, retain=false 2024-12-01T20:24:33,665 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=ea117edd9c2b760fcdfda100b3c4ed5c, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:33,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17330846657683, region=ea117edd9c2b760fcdfda100b3c4ed5c, ASSIGN because future has completed 2024-12-01T20:24:33,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea117edd9c2b760fcdfda100b3c4ed5c, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-01T20:24:33,827 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:24:33,827 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => ea117edd9c2b760fcdfda100b3c4ed5c, NAME => 'ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:33,827 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:24:33,827 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17330846657683 ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,828 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:33,828 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,828 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,830 INFO [StoreOpener-ea117edd9c2b760fcdfda100b3c4ed5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,833 INFO [StoreOpener-ea117edd9c2b760fcdfda100b3c4ed5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea117edd9c2b760fcdfda100b3c4ed5c columnFamilyName f 2024-12-01T20:24:33,833 DEBUG [StoreOpener-ea117edd9c2b760fcdfda100b3c4ed5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:33,834 INFO [StoreOpener-ea117edd9c2b760fcdfda100b3c4ed5c-1 {}] regionserver.HStore(327): Store=ea117edd9c2b760fcdfda100b3c4ed5c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:33,834 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,835 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns4/test-17330846657683/ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,836 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns4/test-17330846657683/ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,836 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,836 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,839 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,842 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns4/test-17330846657683/ea117edd9c2b760fcdfda100b3c4ed5c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:24:33,843 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened ea117edd9c2b760fcdfda100b3c4ed5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10503872160, jitterRate=-0.02175067365169525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:24:33,843 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:24:33,844 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for ea117edd9c2b760fcdfda100b3c4ed5c: Running coprocessor pre-open hook at 1733084673828Writing region info on filesystem at 1733084673828Initializing all the Stores at 1733084673830 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084673830Cleaning up temporary data from old regions at 1733084673836 (+6 ms)Running coprocessor post-open hooks at 1733084673843 (+7 ms)Region opened successfully at 1733084673844 (+1 ms) 2024-12-01T20:24:33,845 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c., pid=19, masterSystemTime=1733084673822 2024-12-01T20:24:33,848 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:24:33,848 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:24:33,849 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=ea117edd9c2b760fcdfda100b3c4ed5c, regionState=OPEN, openSeqNum=2, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:33,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea117edd9c2b760fcdfda100b3c4ed5c, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:24:33,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-01T20:24:33,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure ea117edd9c2b760fcdfda100b3c4ed5c, server=9168bcbe98d9,38993,1733084657496 in 186 msec 2024-12-01T20:24:33,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-01T20:24:33,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17330846657683, region=ea117edd9c2b760fcdfda100b3c4ed5c, ASSIGN in 347 msec 2024-12-01T20:24:33,862 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17330846657683 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:24:33,863 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17330846657683","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084673862"}]},"ts":"1733084673862"} 2024-12-01T20:24:33,877 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17330846657683, state=ENABLED in hbase:meta 2024-12-01T20:24:33,880 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17330846657683 execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:24:33,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17330846657683 in 812 msec 2024-12-01T20:24:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-01T20:24:34,207 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17330846657683 completed 2024-12-01T20:24:34,207 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17330846657683 get assigned. Timeout = 60000ms 2024-12-01T20:24:34,207 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:24:34,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17330846657683 assigned to meta. Checking AM states. 2024-12-01T20:24:34,212 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:24:34,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17330846657683 assigned. 2024-12-01T20:24:34,213 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:24:34,213 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:24:34,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:34,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:34,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:24:34,242 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=150, ProcessCount=18, AvailableMemoryMB=8118 2024-12-01T20:24:34,243 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-12-01T20:24:34,244 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-12-01T20:24:34,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1733084665768 from 'test-1733084665768', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1733084665768', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:24:34,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1733084665768 2024-12-01T20:24:34,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741847_1023 (size=908) 2024-12-01T20:24:34,291 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/.tabledesc/.tableinfo.0000000001.367 2024-12-01T20:24:34,292 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/.tabledesc/.tableinfo.0000000002.908 2024-12-01T20:24:34,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1733084665768}] 2024-12-01T20:24:34,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE}] 2024-12-01T20:24:34,308 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE 2024-12-01T20:24:34,310 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=CLOSING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:34,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE because future has completed 2024-12-01T20:24:34,314 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T20:24:34,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:34,474 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:34,474 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T20:24:34,475 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 00ff3d9048bd8163f6ce55a5b39c9f69, disabling compactions & flushes 2024-12-01T20:24:34,475 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:34,475 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:34,475 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. after waiting 0 ms 2024-12-01T20:24:34,475 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:34,479 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 00ff3d9048bd8163f6ce55a5b39c9f69 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-12-01T20:24:34,548 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f/83cf1f7756204306beca715afde1cb30 is 37, key is row10/f:q1/1733084668473/Put/seqid=0 2024-12-01T20:24:34,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741848_1024 (size=8413) 2024-12-01T20:24:34,967 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f/83cf1f7756204306beca715afde1cb30 2024-12-01T20:24:35,034 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f/83cf1f7756204306beca715afde1cb30 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 2024-12-01T20:24:35,044 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30, entries=99, sequenceid=103, filesize=8.2 K 2024-12-01T20:24:35,051 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 00ff3d9048bd8163f6ce55a5b39c9f69 in 570ms, sequenceid=103, compaction requested=false 2024-12-01T20:24:35,059 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-12-01T20:24:35,063 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:24:35,063 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:35,063 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 00ff3d9048bd8163f6ce55a5b39c9f69: Waiting for close lock at 1733084674475Running coprocessor pre-close hooks at 1733084674475Disabling compacts and flushes for region at 1733084674475Disabling writes for close at 1733084674475Obtaining lock to block concurrent updates at 1733084674479 (+4 ms)Preparing flush snapshotting stores in 00ff3d9048bd8163f6ce55a5b39c9f69 at 1733084674479Finished memstore snapshotting test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1733084674487 (+8 ms)Flushing stores of test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. at 1733084674489 (+2 ms)Flushing 00ff3d9048bd8163f6ce55a5b39c9f69/f: creating writer at 1733084674491 (+2 ms)Flushing 00ff3d9048bd8163f6ce55a5b39c9f69/f: appending metadata at 1733084674536 (+45 ms)Flushing 00ff3d9048bd8163f6ce55a5b39c9f69/f: closing flushed file at 1733084674539 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69fca8c4: reopening flushed file at 1733084675032 (+493 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 00ff3d9048bd8163f6ce55a5b39c9f69 in 570ms, sequenceid=103, compaction requested=false at 1733084675051 (+19 ms)Writing region close event to WAL at 1733084675054 (+3 ms)Running coprocessor post-close hooks at 1733084675060 (+6 ms)Closed at 1733084675063 (+3 ms) 2024-12-01T20:24:35,063 WARN [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 00ff3d9048bd8163f6ce55a5b39c9f69 to self. 2024-12-01T20:24:35,067 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,069 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=CLOSED 2024-12-01T20:24:35,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:24:35,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-01T20:24:35,076 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 in 759 msec 2024-12-01T20:24:35,076 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE; state=CLOSED, location=9168bcbe98d9,38993,1733084657496; forceNewPlan=false, retain=true 2024-12-01T20:24:35,227 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:35,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE because future has completed 2024-12-01T20:24:35,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:35,448 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:35,448 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 00ff3d9048bd8163f6ce55a5b39c9f69, NAME => 'test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:35,448 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:24:35,448 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733084665768 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,448 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:35,448 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,449 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,450 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,452 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ff3d9048bd8163f6ce55a5b39c9f69 columnFamilyName f 2024-12-01T20:24:35,452 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:35,462 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 2024-12-01T20:24:35,462 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(327): Store=00ff3d9048bd8163f6ce55a5b39c9f69/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:35,462 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,464 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ff3d9048bd8163f6ce55a5b39c9f69 columnFamilyName f3 2024-12-01T20:24:35,464 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:35,465 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(327): Store=00ff3d9048bd8163f6ce55a5b39c9f69/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:35,466 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,472 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ff3d9048bd8163f6ce55a5b39c9f69 columnFamilyName mob 2024-12-01T20:24:35,477 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:35,478 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(327): Store=00ff3d9048bd8163f6ce55a5b39c9f69/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:35,479 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,480 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,481 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,483 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,483 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,484 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1733084665768 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T20:24:35,486 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,488 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 00ff3d9048bd8163f6ce55a5b39c9f69; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10376782880, jitterRate=-0.033586785197257996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T20:24:35,488 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:35,489 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 00ff3d9048bd8163f6ce55a5b39c9f69: Running coprocessor pre-open hook at 1733084675449Writing region info on filesystem at 1733084675449Initializing all the Stores at 1733084675450 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084675450Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084675450Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084675450Cleaning up temporary data from old regions at 1733084675483 (+33 ms)Running coprocessor post-open hooks at 1733084675488 (+5 ms)Region opened successfully at 1733084675488 2024-12-01T20:24:35,490 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., pid=24, masterSystemTime=1733084675442 2024-12-01T20:24:35,493 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:35,493 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:35,494 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=OPEN, openSeqNum=107, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:35,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:24:35,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-12-01T20:24:35,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 in 268 msec 2024-12-01T20:24:35,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-01T20:24:35,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE in 1.1950 sec 2024-12-01T20:24:35,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-01T20:24:35,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1733084665768 in 1.2100 sec 2024-12-01T20:24:35,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733084665768 in 1.2610 sec 2024-12-01T20:24:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-12-01T20:24:35,517 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1733084665768 completed 2024-12-01T20:24:35,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acf67ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:35,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:24:35,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:24:35,519 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:24:35,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:24:35,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:24:35,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@381e39ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:35,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:24:35,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:24:35,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:35,521 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:24:35,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c7f766f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:35,523 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:35,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:35,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:35,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:24:35,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733084665768', row='row-test-1733084665768-3-0', locateType=CURRENT is [region=test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., hostname=9168bcbe98d9,38993,1733084657496, seqNum=107] 2024-12-01T20:24:35,578 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T20:24:35,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:35,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ac75734 2024-12-01T20:24:35,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T20:24:35,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T20:24:35,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.3 creating {NAME => 'backup'} 2024-12-01T20:24:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-12-01T20:24:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-01T20:24:35,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 10 msec 2024-12-01T20:24:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-01T20:24:35,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-12-01T20:24:35,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:24:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-12-01T20:24:35,729 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:24:35,729 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:35,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-12-01T20:24:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-01T20:24:35,731 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:24:35,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741849_1025 (size=635) 2024-12-01T20:24:35,759 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8230de2857c5483e5284585a3da8d5f4, NAME => 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:35,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741850_1026 (size=40) 2024-12-01T20:24:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-01T20:24:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-01T20:24:36,169 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:36,170 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 8230de2857c5483e5284585a3da8d5f4, disabling compactions & flushes 2024-12-01T20:24:36,170 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:36,170 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:36,170 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. after waiting 0 ms 2024-12-01T20:24:36,170 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:36,170 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:36,170 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8230de2857c5483e5284585a3da8d5f4: Waiting for close lock at 1733084676170Disabling compacts and flushes for region at 1733084676170Disabling writes for close at 1733084676170Writing region close event to WAL at 1733084676170Closed at 1733084676170 2024-12-01T20:24:36,172 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:24:36,173 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1733084676172"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733084676172"}]},"ts":"1733084676172"} 2024-12-01T20:24:36,176 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:24:36,177 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:24:36,178 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084676178"}]},"ts":"1733084676178"} 2024-12-01T20:24:36,181 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-12-01T20:24:36,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN}] 2024-12-01T20:24:36,183 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN 2024-12-01T20:24:36,184 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN; state=OFFLINE, location=9168bcbe98d9,38993,1733084657496; forceNewPlan=false, retain=false 2024-12-01T20:24:36,263 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17330846657682 2024-12-01T20:24:36,263 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17330846657682 Metrics about Tables on a single HBase RegionServer 2024-12-01T20:24:36,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1733084665768 2024-12-01T20:24:36,264 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1733084665768 Metrics about Tables on a single HBase RegionServer 2024-12-01T20:24:36,265 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17330846657683 2024-12-01T20:24:36,265 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17330846657683 Metrics about Tables on a single HBase RegionServer 2024-12-01T20:24:36,265 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17330846657681 2024-12-01T20:24:36,266 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17330846657681 Metrics about Tables on a single HBase RegionServer 2024-12-01T20:24:36,335 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8230de2857c5483e5284585a3da8d5f4, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:36,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN because future has completed 2024-12-01T20:24:36,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-01T20:24:36,496 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:36,497 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 8230de2857c5483e5284585a3da8d5f4, NAME => 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:36,497 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:24:36,497 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,497 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:36,497 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,497 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,500 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,502 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8230de2857c5483e5284585a3da8d5f4 columnFamilyName meta 2024-12-01T20:24:36,502 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:36,503 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.HStore(327): Store=8230de2857c5483e5284585a3da8d5f4/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:36,504 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,506 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8230de2857c5483e5284585a3da8d5f4 columnFamilyName session 2024-12-01T20:24:36,506 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:36,508 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.HStore(327): Store=8230de2857c5483e5284585a3da8d5f4/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:36,508 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,509 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,510 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,511 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,511 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,512 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T20:24:36,514 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,517 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:24:36,518 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 8230de2857c5483e5284585a3da8d5f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10752699840, jitterRate=0.0014232099056243896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T20:24:36,518 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:36,519 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 8230de2857c5483e5284585a3da8d5f4: Running coprocessor pre-open hook at 1733084676498Writing region info on filesystem at 1733084676498Initializing all the Stores at 1733084676499 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084676499Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084676500 (+1 ms)Cleaning up temporary data from old regions at 1733084676511 (+11 ms)Running coprocessor post-open hooks at 1733084676518 (+7 ms)Region opened successfully at 1733084676519 (+1 ms) 2024-12-01T20:24:36,520 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., pid=28, masterSystemTime=1733084676491 2024-12-01T20:24:36,522 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:36,523 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:36,523 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=8230de2857c5483e5284585a3da8d5f4, regionState=OPEN, openSeqNum=2, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:36,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:24:36,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-01T20:24:36,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496 in 188 msec 2024-12-01T20:24:36,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-01T20:24:36,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN in 349 msec 2024-12-01T20:24:36,535 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:24:36,535 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084676535"}]},"ts":"1733084676535"} 2024-12-01T20:24:36,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-12-01T20:24:36,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:24:36,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 813 msec 2024-12-01T20:24:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-01T20:24:36,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-12-01T20:24:36,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:24:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-12-01T20:24:36,888 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:24:36,888 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:36,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-12-01T20:24:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-01T20:24:36,890 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:24:36,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741851_1027 (size=640) 2024-12-01T20:24:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-01T20:24:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-01T20:24:37,302 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c27f3c1374897bd332c0d1fd4656bf72, NAME => 'backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:37,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741852_1028 (size=45) 2024-12-01T20:24:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-01T20:24:37,717 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:37,717 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing c27f3c1374897bd332c0d1fd4656bf72, disabling compactions & flushes 2024-12-01T20:24:37,717 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:24:37,717 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:24:37,717 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. after waiting 0 ms 2024-12-01T20:24:37,717 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:24:37,717 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:24:37,717 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for c27f3c1374897bd332c0d1fd4656bf72: Waiting for close lock at 1733084677717Disabling compacts and flushes for region at 1733084677717Disabling writes for close at 1733084677717Writing region close event to WAL at 1733084677717Closed at 1733084677717 2024-12-01T20:24:37,720 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:24:37,720 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733084677720"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733084677720"}]},"ts":"1733084677720"} 2024-12-01T20:24:37,725 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:24:37,727 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:24:37,727 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084677727"}]},"ts":"1733084677727"} 2024-12-01T20:24:37,730 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-12-01T20:24:37,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=c27f3c1374897bd332c0d1fd4656bf72, ASSIGN}] 2024-12-01T20:24:37,731 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=c27f3c1374897bd332c0d1fd4656bf72, ASSIGN 2024-12-01T20:24:37,733 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=c27f3c1374897bd332c0d1fd4656bf72, ASSIGN; state=OFFLINE, location=9168bcbe98d9,38993,1733084657496; forceNewPlan=false, retain=false 2024-12-01T20:24:37,883 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c27f3c1374897bd332c0d1fd4656bf72, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:37,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=c27f3c1374897bd332c0d1fd4656bf72, ASSIGN because future has completed 2024-12-01T20:24:37,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c27f3c1374897bd332c0d1fd4656bf72, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:24:38,026 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17330846657682' 2024-12-01T20:24:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-01T20:24:38,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17330846657683' 2024-12-01T20:24:38,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-12-01T20:24:38,044 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:24:38,044 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => c27f3c1374897bd332c0d1fd4656bf72, NAME => 'backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:24:38,045 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:24:38,045 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,045 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:24:38,045 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,045 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,047 INFO [StoreOpener-c27f3c1374897bd332c0d1fd4656bf72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,048 INFO [StoreOpener-c27f3c1374897bd332c0d1fd4656bf72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c27f3c1374897bd332c0d1fd4656bf72 columnFamilyName meta 2024-12-01T20:24:38,048 DEBUG [StoreOpener-c27f3c1374897bd332c0d1fd4656bf72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:38,049 INFO [StoreOpener-c27f3c1374897bd332c0d1fd4656bf72-1 {}] regionserver.HStore(327): Store=c27f3c1374897bd332c0d1fd4656bf72/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:38,049 INFO [StoreOpener-c27f3c1374897bd332c0d1fd4656bf72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,051 INFO [StoreOpener-c27f3c1374897bd332c0d1fd4656bf72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c27f3c1374897bd332c0d1fd4656bf72 columnFamilyName session 2024-12-01T20:24:38,051 DEBUG [StoreOpener-c27f3c1374897bd332c0d1fd4656bf72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:38,051 INFO [StoreOpener-c27f3c1374897bd332c0d1fd4656bf72-1 {}] regionserver.HStore(327): Store=c27f3c1374897bd332c0d1fd4656bf72/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:24:38,051 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,052 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system_bulk/c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,053 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system_bulk/c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,054 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,054 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,055 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T20:24:38,057 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,060 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system_bulk/c27f3c1374897bd332c0d1fd4656bf72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:24:38,061 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened c27f3c1374897bd332c0d1fd4656bf72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9655481120, jitterRate=-0.1007632464170456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T20:24:38,061 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:24:38,062 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for c27f3c1374897bd332c0d1fd4656bf72: Running coprocessor pre-open hook at 1733084678045Writing region info on filesystem at 1733084678045Initializing all the Stores at 1733084678046 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084678046Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084678047 (+1 ms)Cleaning up temporary data from old regions at 1733084678054 (+7 ms)Running coprocessor post-open hooks at 1733084678061 (+7 ms)Region opened successfully at 1733084678062 (+1 ms) 2024-12-01T20:24:38,063 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72., pid=31, masterSystemTime=1733084678039 2024-12-01T20:24:38,066 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:24:38,066 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:24:38,067 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c27f3c1374897bd332c0d1fd4656bf72, regionState=OPEN, openSeqNum=2, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:38,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c27f3c1374897bd332c0d1fd4656bf72, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:24:38,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-01T20:24:38,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure c27f3c1374897bd332c0d1fd4656bf72, server=9168bcbe98d9,38993,1733084657496 in 185 msec 2024-12-01T20:24:38,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-01T20:24:38,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=c27f3c1374897bd332c0d1fd4656bf72, ASSIGN in 343 msec 2024-12-01T20:24:38,077 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:24:38,077 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084678077"}]},"ts":"1733084678077"} 2024-12-01T20:24:38,079 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-12-01T20:24:38,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:24:38,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.1950 sec 2024-12-01T20:24:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-01T20:24:39,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-12-01T20:24:39,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:24:39,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:24:39,067 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-01T20:24:39,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., hostname=9168bcbe98d9,38993,1733084657496, seqNum=2] 2024-12-01T20:24:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38993 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-12-01T20:24:39,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:24:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084679090 (current time:1733084679090). 2024-12-01T20:24:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:24:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-01T20:24:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3742f5e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:24:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:24:39,092 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:24:39,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:24:39,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:24:39,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e7528d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:39,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:24:39,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:24:39,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:39,094 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:24:39,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54386e52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:39,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:39,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:39,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:24:39,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:24:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:24:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:39,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:24:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:24:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:24:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-12-01T20:24:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-01T20:24:39,112 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:24:39,117 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:24:39,130 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:24:39,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741853_1029 (size=52) 2024-12-01T20:24:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-01T20:24:39,330 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-01T20:24:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-01T20:24:39,539 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:24:39,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4}] 2024-12-01T20:24:39,548 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:39,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-12-01T20:24:39,705 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:39,706 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 8230de2857c5483e5284585a3da8d5f4 2/2 column families, dataSize=45 B heapSize=632 B 2024-12-01T20:24:39,723 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/0bf3de11a9904d678102c77d02ef03f8 is 49, key is activesession:/session:c/1733084679082/Put/seqid=0 2024-12-01T20:24:39,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741854_1030 (size=5049) 2024-12-01T20:24:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-01T20:24:40,130 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/0bf3de11a9904d678102c77d02ef03f8 2024-12-01T20:24:40,147 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/0bf3de11a9904d678102c77d02ef03f8 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 2024-12-01T20:24:40,157 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8, entries=1, sequenceid=5, filesize=4.9 K 2024-12-01T20:24:40,158 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 8230de2857c5483e5284585a3da8d5f4 in 452ms, sequenceid=5, compaction requested=false 2024-12-01T20:24:40,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 8230de2857c5483e5284585a3da8d5f4: 2024-12-01T20:24:40,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. for snapshot_backup_system completed. 2024-12-01T20:24:40,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.' region-info for snapshot=snapshot_backup_system 2024-12-01T20:24:40,163 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:24:40,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-01T20:24:40,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8] hfiles 2024-12-01T20:24:40,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 for snapshot=snapshot_backup_system 2024-12-01T20:24:40,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741855_1031 (size=95) 2024-12-01T20:24:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-01T20:24:40,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:40,581 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-01T20:24:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-01T20:24:40,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:40,584 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:40,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-01T20:24:40,645 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:24:40,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 in 1.0990 sec 2024-12-01T20:24:40,647 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:24:40,649 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:24:40,649 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-01T20:24:40,653 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-01T20:24:40,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741856_1032 (size=731) 2024-12-01T20:24:41,076 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:24:41,088 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:24:41,089 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_backup_system 2024-12-01T20:24:41,091 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:24:41,091 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-12-01T20:24:41,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9840 sec 2024-12-01T20:24:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-01T20:24:41,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-01T20:24:41,260 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733084675563 started at 1733084681258. 2024-12-01T20:24:41,280 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733084675563 has been started. 2024-12-01T20:24:41,284 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-12-01T20:24:41,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-01T20:24:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-01T20:24:41,292 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-01T20:24:41,292 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-01T20:24:41,293 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-01T20:24:41,293 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-01T20:24:41,294 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,294 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-01T20:24:41,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-01T20:24:41,382 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,382 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-01T20:24:41,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:24:41,382 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,383 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-01T20:24:41,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-01T20:24:41,384 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-01T20:24:41,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-01T20:24:41,386 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-01T20:24:41,389 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-01T20:24:41,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-01T20:24:41,390 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-01T20:24:41,391 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-01T20:24:41,391 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-01T20:24:41,391 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-01T20:24:41,391 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '9168bcbe98d9,38993,1733084657496' joining acquired barrier for procedure (rolllog) in zk 2024-12-01T20:24:41,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,404 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-01T20:24:41,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:24:41,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:24:41,405 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-01T20:24:41,405 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-01T20:24:41,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:24:41,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:24:41,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:24:41,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:24:41,409 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '9168bcbe98d9,38993,1733084657496' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-01T20:24:41,409 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-01T20:24:41,409 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@560e935c[Count = 0] remaining members to acquire global barrier 2024-12-01T20:24:41,409 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-01T20:24:41,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:24:41,414 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-01T20:24:41,415 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-01T20:24:41,415 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-01T20:24:41,415 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,415 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-01T20:24:41,416 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-01T20:24:41,416 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,422 INFO [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733084660201 highest: 1733084667472 on 9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,422 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.meta:.meta(num 1733084661072) roll requested 2024-12-01T20:24:41,435 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta, exclude list is [], retry=0 2024-12-01T20:24:41,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:24:41,441 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084661072.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta 2024-12-01T20:24:41,442 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:24:41,442 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084661072.meta is not closed yet, will try archiving it next time 2024-12-01T20:24:41,442 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1:(num 1733084667472) roll requested 2024-12-01T20:24:41,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741834_1010 (size=12055) 2024-12-01T20:24:41,454 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442, exclude list is [], retry=0 2024-12-01T20:24:41,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:24:41,459 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084667472 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 2024-12-01T20:24:41,459 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:24:41,459 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084667472 is not closed yet, will try archiving it next time 2024-12-01T20:24:41,459 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0:(num 1733084660201) roll requested 2024-12-01T20:24:41,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741840_1016 (size=4756) 2024-12-01T20:24:41,461 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084667472 not finished, retry = 0 2024-12-01T20:24:41,471 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459, exclude list is [], retry=0 2024-12-01T20:24:41,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:24:41,475 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084660201 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 2024-12-01T20:24:41,475 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:24:41,476 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084660201 is not closed yet, will try archiving it next time 2024-12-01T20:24:41,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741833_1009 (size=1034) 2024-12-01T20:24:41,477 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084660201 not finished, retry = 0 2024-12-01T20:24:41,482 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-01T20:24:41,483 INFO [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733084681459 on 9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,483 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T20:24:41,483 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is 9168bcbe98d9,39089,1733084655909 2024-12-01T20:24:41,483 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1be37a59 2024-12-01T20:24:41,483 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T20:24:41,484 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53551, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-01T20:24:41,485 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:41,485 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:41,485 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:41,486 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55419, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-01T20:24:41,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:24:41,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:24:41,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:44065/backupUT\x00', locateType=CURRENT is [region=backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., hostname=9168bcbe98d9,38993,1733084657496, seqNum=2] 2024-12-01T20:24:41,510 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-01T20:24:41,511 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '9168bcbe98d9,38993,1733084657496' in zk 2024-12-01T20:24:41,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,520 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-01T20:24:41,520 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-01T20:24:41,520 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-01T20:24:41,520 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,520 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:24:41,520 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:24:41,521 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:24:41,521 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:24:41,522 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:24:41,522 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,523 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:24:41,523 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:24:41,523 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,524 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '9168bcbe98d9,38993,1733084657496': 2024-12-01T20:24:41,524 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-01T20:24:41,524 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-01T20:24:41,524 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-01T20:24:41,524 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-01T20:24:41,524 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '9168bcbe98d9,38993,1733084657496' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-01T20:24:41,525 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-01T20:24:41,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-01T20:24:41,530 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,530 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,530 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:24:41,530 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,530 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:24:41,531 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:24:41,531 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-01T20:24:41,531 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,531 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:24:41,531 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:24:41,531 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,532 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:24:41,532 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:24:41,532 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,532 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:24:41,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:24:41,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:41,550 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:24:41,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:24:41,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:24:41,550 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-01T20:24:41,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-01T20:24:41,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-01T20:24:41,551 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-01T20:24:41,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:24:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-01T20:24:41,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } 2024-12-01T20:24:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084681563 (current time:1733084681563). 2024-12-01T20:24:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:24:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733084681562_default_test-1733084665768 VERSION not specified, setting to 2 2024-12-01T20:24:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c34e152, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:24:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:24:41,565 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:24:41,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:24:41,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:24:41,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eba9714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:41,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:24:41,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:24:41,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:41,567 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:24:41,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4018955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:41,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:41,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:41,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:24:41,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:24:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:24:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:41,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:24:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:24:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } 2024-12-01T20:24:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-12-01T20:24:41,575 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:24:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-01T20:24:41,576 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:24:41,579 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084660201 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084660201 2024-12-01T20:24:41,579 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:24:41,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741860_1036 (size=84) 2024-12-01T20:24:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-01T20:24:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-01T20:24:41,991 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:24:41,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69}] 2024-12-01T20:24:41,995 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:42,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-12-01T20:24:42,151 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:42,152 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 00ff3d9048bd8163f6ce55a5b39c9f69 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-12-01T20:24:42,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f3/d5889c07403849ceb7069c51feadb85f is 58, key is row-test-1733084665768-3-0/f3:q1/1733084675531/Put/seqid=0 2024-12-01T20:24:42,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741861_1037 (size=5387) 2024-12-01T20:24:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-01T20:24:42,583 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f3/d5889c07403849ceb7069c51feadb85f 2024-12-01T20:24:42,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741862_1038 (size=4469) 2024-12-01T20:24:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-01T20:24:43,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/mob/c1f501fd15fc4f31aff54a45b93dcedf is 59, key is row-test-1733084665768-3-0/mob:q1/1733084675546/Put/seqid=0 2024-12-01T20:24:43,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741863_1039 (size=5484) 2024-12-01T20:24:43,435 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/mob/c1f501fd15fc4f31aff54a45b93dcedf 2024-12-01T20:24:43,443 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f3/d5889c07403849ceb7069c51feadb85f as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f 2024-12-01T20:24:43,451 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f, entries=6, sequenceid=121, filesize=5.3 K 2024-12-01T20:24:43,452 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/mob/c1f501fd15fc4f31aff54a45b93dcedf as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf 2024-12-01T20:24:43,459 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf, entries=6, sequenceid=121, filesize=5.4 K 2024-12-01T20:24:43,460 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 00ff3d9048bd8163f6ce55a5b39c9f69 in 1309ms, sequenceid=121, compaction requested=false 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 00ff3d9048bd8163f6ce55a5b39c9f69: 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. for snapshot_1733084681562_default_test-1733084665768 completed. 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.' region-info for snapshot=snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30] hfiles 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 for snapshot=snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f] hfiles 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f for snapshot=snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf] hfiles 2024-12-01T20:24:43,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf for snapshot=snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:43,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741864_1040 (size=185) 2024-12-01T20:24:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-01T20:24:43,868 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:43,868 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-01T20:24:43,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-01T20:24:43,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733084681562_default_test-1733084665768 on region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:43,869 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:43,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-01T20:24:43,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69 in 1.8780 sec 2024-12-01T20:24:43,873 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:24:43,874 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:24:43,876 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-01T20:24:43,876 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-01T20:24:43,877 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:43,878 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-12-01T20:24:43,878 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:43,879 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-12-01T20:24:43,879 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:43,879 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-12-01T20:24:43,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741865_1041 (size=49) 2024-12-01T20:24:44,033 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-12-01T20:24:44,290 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:24:44,290 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:44,292 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:44,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741866_1042 (size=1146) 2024-12-01T20:24:44,716 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:24:44,725 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:24:44,726 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084681562_default_test-1733084665768 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:44,728 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:24:44,728 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-12-01T20:24:44,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733084681562_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } in 3.1550 sec 2024-12-01T20:24:44,992 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T20:24:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-01T20:24:45,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1733084665768 completed 2024-12-01T20:24:45,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } 2024-12-01T20:24:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084685751 (current time:1733084685751). 2024-12-01T20:24:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:24:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733084685749_ns2_test-17330846657681 VERSION not specified, setting to 2 2024-12-01T20:24:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28f3fe2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:24:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:24:45,756 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:24:45,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:24:45,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:24:45,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d502d28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:45,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:24:45,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:24:45,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:45,760 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:24:45,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc8802f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:45,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:45,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:45,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:24:45,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:24:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:24:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:45,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:24:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:24:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } 2024-12-01T20:24:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-12-01T20:24:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-01T20:24:45,770 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:24:45,771 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:24:45,832 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:24:45,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741867_1043 (size=86) 2024-12-01T20:24:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-01T20:24:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-01T20:24:46,240 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:24:46,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a3c8df032a6e62b8e874b176ca842b1a}] 2024-12-01T20:24:46,242 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:46,263 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-12-01T20:24:46,263 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-12-01T20:24:46,264 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-12-01T20:24:46,264 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-12-01T20:24:46,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-12-01T20:24:46,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:46,396 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing a3c8df032a6e62b8e874b176ca842b1a 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-12-01T20:24:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-01T20:24:46,416 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/.tmp/f/c527d074387247ea8c82527b03b9793e is 37, key is row10/f:q1/1733084670805/Put/seqid=0 2024-12-01T20:24:46,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741868_1044 (size=8413) 2024-12-01T20:24:46,822 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/.tmp/f/c527d074387247ea8c82527b03b9793e 2024-12-01T20:24:46,838 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/.tmp/f/c527d074387247ea8c82527b03b9793e as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e 2024-12-01T20:24:46,847 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e, entries=99, sequenceid=103, filesize=8.2 K 2024-12-01T20:24:46,848 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for a3c8df032a6e62b8e874b176ca842b1a in 452ms, sequenceid=103, compaction requested=false 2024-12-01T20:24:46,848 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for a3c8df032a6e62b8e874b176ca842b1a: 2024-12-01T20:24:46,848 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. for snapshot_1733084685749_ns2_test-17330846657681 completed. 2024-12-01T20:24:46,848 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a.' region-info for snapshot=snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:46,849 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:24:46,849 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e] hfiles 2024-12-01T20:24:46,849 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e for snapshot=snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:46,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741869_1045 (size=91) 2024-12-01T20:24:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-01T20:24:47,258 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:24:47,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-01T20:24:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-01T20:24:47,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733084685749_ns2_test-17330846657681 on region a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:47,262 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:24:47,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-01T20:24:47,270 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:24:47,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a3c8df032a6e62b8e874b176ca842b1a in 1.0250 sec 2024-12-01T20:24:47,271 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:24:47,272 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:24:47,272 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:47,273 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:47,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741870_1046 (size=456) 2024-12-01T20:24:47,688 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:24:47,699 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:24:47,699 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084685749_ns2_test-17330846657681 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:47,701 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:24:47,701 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-12-01T20:24:47,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733084685749_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } in 1.9340 sec 2024-12-01T20:24:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-01T20:24:47,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17330846657681 completed 2024-12-01T20:24:47,919 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1733084675563 2024-12-01T20:24:47,919 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-12-01T20:24:47,921 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-12-01T20:24:47,924 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733084675563_ns2:test-17330846657681 2024-12-01T20:24:47,925 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733084685749_ns2_test-17330846657681 to hdfs://localhost:44065/backupUT/backup_1733084675563/ns2/test-17330846657681/ 2024-12-01T20:24:47,930 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-01T20:24:47,961 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44065, inputRoot=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:47,961 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-454178622_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44065/backupUT/backup_1733084675563/ns2/test-17330846657681, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44065/backupUT/backup_1733084675563/ns2/test-17330846657681/.hbase-snapshot/.tmp/snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:47,963 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-01T20:24:47,969 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084685749_ns2_test-17330846657681 to hdfs://localhost:44065/backupUT/backup_1733084675563/ns2/test-17330846657681/.hbase-snapshot/.tmp/snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:47,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741871_1047 (size=86) 2024-12-01T20:24:47,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741872_1048 (size=456) 2024-12-01T20:24:48,392 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:48,393 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:48,393 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,198 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-11443089343952062177.jar 2024-12-01T20:24:49,198 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,199 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,199 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,199 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,199 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,200 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,200 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,200 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:49,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:24:49,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:24:49,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:24:49,201 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:24:49,202 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:24:49,202 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:24:49,202 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:24:49,202 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:24:49,203 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:24:49,203 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:24:49,203 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:24:49,205 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:24:49,205 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:24:49,205 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:24:49,205 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:24:49,206 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:24:49,206 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:24:49,206 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:24:49,259 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:24:49,286 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733084685749_ns2_test-17330846657681' hfile list 2024-12-01T20:24:49,291 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-12-01T20:24:49,485 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local1554221820_0001_f13c664d-421d-480b-b7dd-e2a62426c7ea/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-01T20:24:49,486 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local1554221820_0001_f13c664d-421d-480b-b7dd-e2a62426c7ea/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-01T20:24:49,656 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-01T20:24:49,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741873_1049 (size=8413) 2024-12-01T20:24:50,143 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e output=hdfs://localhost:44065/backupUT/backup_1733084675563/ns2/test-17330846657681/archive/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e 2024-12-01T20:24:50,145 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.017M/sec 2024-12-01T20:24:50,580 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-01T20:24:50,581 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-01T20:24:50,588 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:50,588 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733084685749_ns2_test-17330846657681 finished. 2024-12-01T20:24:50,588 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733084675563_test-1733084665768 2024-12-01T20:24:50,588 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733084681562_default_test-1733084665768 to hdfs://localhost:44065/backupUT/backup_1733084675563/default/test-1733084665768/ 2024-12-01T20:24:50,588 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-01T20:24:50,616 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44065, inputRoot=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:24:50,616 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-454178622_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44065/backupUT/backup_1733084675563/default/test-1733084665768, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44065/backupUT/backup_1733084675563/default/test-1733084665768/.hbase-snapshot/.tmp/snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:50,618 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-01T20:24:50,625 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084681562_default_test-1733084665768 to hdfs://localhost:44065/backupUT/backup_1733084675563/default/test-1733084665768/.hbase-snapshot/.tmp/snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741874_1050 (size=1146) 2024-12-01T20:24:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741875_1051 (size=84) 2024-12-01T20:24:51,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,838 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-1564734101742128570.jar 2024-12-01T20:24:51,838 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,839 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,839 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,839 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,839 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,839 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,840 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,840 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,840 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:24:51,840 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:24:51,840 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:24:51,841 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:24:51,841 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:24:51,841 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:24:51,841 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:24:51,841 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:24:51,842 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:24:51,842 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:24:51,842 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:24:51,842 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:24:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:24:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:24:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:24:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:24:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:24:51,843 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:24:51,844 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:24:51,878 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:24:51,901 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733084681562_default_test-1733084665768' hfile list 2024-12-01T20:24:51,904 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-12-01T20:24:51,982 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local1479642950_0002_33d64988-b346-49c8-99a4-2408413944cc/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-01T20:24:51,982 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local1479642950_0002_33d64988-b346-49c8-99a4-2408413944cc/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-01T20:24:52,062 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-01T20:24:52,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741876_1052 (size=8413) 2024-12-01T20:24:52,500 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 output=hdfs://localhost:44065/backupUT/backup_1733084675563/default/test-1733084665768/archive/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 2024-12-01T20:24:52,500 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-12-01T20:24:52,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741877_1053 (size=5484) 2024-12-01T20:24:52,960 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf output=hdfs://localhost:44065/backupUT/backup_1733084675563/default/test-1733084665768/archive/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf 2024-12-01T20:24:52,960 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-12-01T20:24:53,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741878_1054 (size=5387) 2024-12-01T20:24:53,407 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f output=hdfs://localhost:44065/backupUT/backup_1733084675563/default/test-1733084665768/archive/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f 2024-12-01T20:24:53,407 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.012M/sec 2024-12-01T20:24:54,037 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-01T20:24:54,038 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-01T20:24:54,050 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:54,050 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733084681562_default_test-1733084665768 finished. 2024-12-01T20:24:54,050 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17330846657681 2024-12-01T20:24:54,050 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1733084665768 2024-12-01T20:24:54,094 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733084675563 2024-12-01T20:24:54,094 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-12-01T20:24:54,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741879_1055 (size=130) 2024-12-01T20:24:54,504 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:44065/backupUT/backup_1733084675563/.backup.manifest 2024-12-01T20:24:54,510 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733084675563 finished: type=FULL,tablelist=ns2:test-17330846657681;test-1733084665768,targetRootDir=hdfs://localhost:44065/backupUT,startts=1733084681258,completets=1733084694092,bytescopied=0 2024-12-01T20:24:54,510 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-12-01T20:24:54,511 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:54,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733084685749_ns2_test-17330846657681" type: DISABLED 2024-12-01T20:24:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733084685749_ns2_test-17330846657681 2024-12-01T20:24:54,521 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733084685749_ns2_test-17330846657681 for backup backup_1733084675563 succeeded. 2024-12-01T20:24:54,521 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:54,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733084681562_default_test-1733084665768" type: DISABLED 2024-12-01T20:24:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733084681562_default_test-1733084665768 2024-12-01T20:24:54,525 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733084681562_default_test-1733084665768 for backup backup_1733084675563 succeeded. 2024-12-01T20:24:54,526 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-01T20:24:54,536 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-01T20:24:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-01T20:24:54,540 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-01T20:24:54,546 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-01T20:24:54,560 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733084675563 completed. 2024-12-01T20:24:54,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:24:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:24:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1733084675563', locateType=CURRENT is [region=backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., hostname=9168bcbe98d9,38993,1733084657496, seqNum=2] 2024-12-01T20:24:54,759 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1733084665768 2024-12-01T20:24:54,775 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1733084665768 2024-12-01T20:24:54,949 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1733084665768 to Mob enabled CF 2024-12-01T20:24:54,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17330846657681', row='row-t20', locateType=CURRENT is [region=ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a., hostname=9168bcbe98d9,38993,1733084657496, seqNum=2] 2024-12-01T20:24:54,988 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17330846657681 2024-12-01T20:24:54,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.3 split test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:24:54,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] assignment.SplitTableRegionProcedure(210): test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. splitKey isn't explicitly specified, will try to find a best split key from RS 9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:55,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 because midkey is the same as first or last row 2024-12-01T20:24:55,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f because midkey is the same as first or last row 2024-12-01T20:24:55,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf because midkey is the same as first or last row 2024-12-01T20:24:55,001 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.HRegion(2902): Flushing 00ff3d9048bd8163f6ce55a5b39c9f69 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-12-01T20:24:55,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f/7d5ed29d1a5a4760bc41d5b043afaf11 is 59, key is row-test-1733084665768-1-10/f:q1/1733084694607/Put/seqid=0 2024-12-01T20:24:55,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741880_1056 (size=10796) 2024-12-01T20:24:55,427 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f/7d5ed29d1a5a4760bc41d5b043afaf11 2024-12-01T20:24:55,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69 is 63, key is row-test-1733084665768-3-100/mob:q1/1733084694933/Put/seqid=0 2024-12-01T20:24:55,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741881_1057 (size=5591) 2024-12-01T20:24:55,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:24:55,869 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/mobdir/data/default/test-1733084665768/7d794a83a2f4a86ecdb78d0f8ec3c014/mob/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:24:55,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/mob/ec20565cf6cf4889a92097059cb26fc4, store: [table=test-1733084665768 family=mob region=00ff3d9048bd8163f6ce55a5b39c9f69] 2024-12-01T20:24:55,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/mob/ec20565cf6cf4889a92097059cb26fc4 is 192, key is row-test-1733084665768-3-100/mob:q1/1733084694933/Put/seqid=0 2024-12-01T20:24:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741882_1058 (size=13760) 2024-12-01T20:24:56,276 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/mob/ec20565cf6cf4889a92097059cb26fc4 2024-12-01T20:24:56,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f/7d5ed29d1a5a4760bc41d5b043afaf11 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/7d5ed29d1a5a4760bc41d5b043afaf11 2024-12-01T20:24:56,291 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/7d5ed29d1a5a4760bc41d5b043afaf11, entries=99, sequenceid=334, filesize=10.5 K 2024-12-01T20:24:56,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/mob/ec20565cf6cf4889a92097059cb26fc4 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/ec20565cf6cf4889a92097059cb26fc4 2024-12-01T20:24:56,299 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/ec20565cf6cf4889a92097059cb26fc4, entries=111, sequenceid=334, filesize=13.4 K 2024-12-01T20:24:56,300 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 00ff3d9048bd8163f6ce55a5b39c9f69 in 1300ms, sequenceid=334, compaction requested=false 2024-12-01T20:24:56,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.HRegion(2603): Flush status journal for 00ff3d9048bd8163f6ce55a5b39c9f69: 2024-12-01T20:24:56,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/7d5ed29d1a5a4760bc41d5b043afaf11 because midkey is the same as first or last row 2024-12-01T20:24:56,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f because midkey is the same as first or last row 2024-12-01T20:24:56,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/ec20565cf6cf4889a92097059cb26fc4 because midkey is the same as first or last row 2024-12-01T20:24:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=9168bcbe98d9,38993,1733084657496 2024-12-01T20:24:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-01T20:24:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.CallRunner(138): callId: 302 service: MasterService methodName: SplitRegion size: 98 connection: 172.17.0.3:36938 deadline: 1733084754995, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-12-01T20:24:56,307 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:24:56,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1733084665768 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:24:56,308 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:24:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733084665768,, stopping at row=test-1733084665768 ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,319 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733084665768,, stopping at row=test-1733084665768 ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,327 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1339 2024-12-01T20:24:56,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,369 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-01T20:24:56,373 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733084696327 2024-12-01T20:24:56,379 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733084675563} 2024-12-01T20:24:56,379 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-01T20:24:56,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:24:56,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:24:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084696426 (current time:1733084696426). 2024-12-01T20:24:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:24:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-01T20:24:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af1711c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:24:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:24:56,428 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:24:56,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:24:56,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:24:56,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2caae6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:56,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:24:56,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:24:56,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:56,430 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:24:56,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c6154ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:24:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:24:56,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:24:56,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:24:56,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:24:56,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:24:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:24:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:24:56,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:24:56,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:24:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:24:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-12-01T20:24:56,438 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:24:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-01T20:24:56,439 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:24:56,442 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:24:56,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741883_1059 (size=52) 2024-12-01T20:24:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-01T20:24:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-01T20:24:56,850 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:24:56,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4}] 2024-12-01T20:24:56,852 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:57,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-12-01T20:24:57,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:57,007 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 8230de2857c5483e5284585a3da8d5f4 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-12-01T20:24:57,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/56198cfac43b4c6e967ddfbb9ad053cc is 158, key is trslm:hdfs://localhost:44065/backupUT\x00ns2:test-17330846657681/meta:log-roll-map/1733084694082/Put/seqid=0 2024-12-01T20:24:57,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741884_1060 (size=5914) 2024-12-01T20:24:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-01T20:24:57,436 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/56198cfac43b4c6e967ddfbb9ad053cc 2024-12-01T20:24:57,451 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/253b2be5b03848e5b74c1bafdfce328e is 433, key is session:backup_1733084675563/session:context/1733084694544/Put/seqid=0 2024-12-01T20:24:57,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741885_1061 (size=5535) 2024-12-01T20:24:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-01T20:24:57,857 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/253b2be5b03848e5b74c1bafdfce328e 2024-12-01T20:24:57,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/56198cfac43b4c6e967ddfbb9ad053cc as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc 2024-12-01T20:24:57,882 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc, entries=6, sequenceid=17, filesize=5.8 K 2024-12-01T20:24:57,883 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/253b2be5b03848e5b74c1bafdfce328e as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e 2024-12-01T20:24:57,890 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e, entries=2, sequenceid=17, filesize=5.4 K 2024-12-01T20:24:57,891 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 8230de2857c5483e5284585a3da8d5f4 in 884ms, sequenceid=17, compaction requested=false 2024-12-01T20:24:57,891 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 8230de2857c5483e5284585a3da8d5f4: 2024-12-01T20:24:57,891 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. for snapshot_backup_system completed. 2024-12-01T20:24:57,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.' region-info for snapshot=snapshot_backup_system 2024-12-01T20:24:57,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:24:57,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc] hfiles 2024-12-01T20:24:57,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc for snapshot=snapshot_backup_system 2024-12-01T20:24:57,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e] hfiles 2024-12-01T20:24:57,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 for snapshot=snapshot_backup_system 2024-12-01T20:24:57,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e for snapshot=snapshot_backup_system 2024-12-01T20:24:57,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741886_1062 (size=173) 2024-12-01T20:24:58,300 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:24:58,300 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-12-01T20:24:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-12-01T20:24:58,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:58,303 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:24:58,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-01T20:24:58,311 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:24:58,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 in 1.4560 sec 2024-12-01T20:24:58,312 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:24:58,313 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:24:58,313 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-01T20:24:58,314 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-01T20:24:58,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741887_1063 (size=810) 2024-12-01T20:24:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-01T20:24:58,731 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:24:58,741 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:24:58,742 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_backup_system 2024-12-01T20:24:58,743 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:24:58,743 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-12-01T20:24:58,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3070 sec 2024-12-01T20:25:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-01T20:25:00,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-01T20:25:00,608 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733084696327 started at 1733084700608. 2024-12-01T20:25:00,614 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733084696327 has been started. 2024-12-01T20:25:00,622 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17330846657681, test-1733084665768] 2024-12-01T20:25:00,628 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1733084667472for backupID backup_1733084696327 2024-12-01T20:25:00,628 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-12-01T20:25:00,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-01T20:25:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-01T20:25:00,629 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-01T20:25:00,630 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-01T20:25:00,630 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-01T20:25:00,630 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-01T20:25:00,630 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,630 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:00,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-01T20:25:00,698 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,698 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-01T20:25:00,698 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:25:00,699 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,700 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-01T20:25:00,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:00,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-01T20:25:00,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:00,703 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-01T20:25:00,704 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-01T20:25:00,704 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-01T20:25:00,704 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-01T20:25:00,704 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-01T20:25:00,704 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-01T20:25:00,704 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-01T20:25:00,704 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '9168bcbe98d9,38993,1733084657496' joining acquired barrier for procedure (rolllog) in zk 2024-12-01T20:25:00,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,718 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:00,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:00,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:00,719 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-01T20:25:00,719 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-01T20:25:00,720 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:00,721 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:00,722 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:00,723 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,724 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:00,725 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '9168bcbe98d9,38993,1733084657496' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-01T20:25:00,725 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5c5e08bc[Count = 0] remaining members to acquire global barrier 2024-12-01T20:25:00,725 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-01T20:25:00,725 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:00,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:00,739 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:00,739 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:00,739 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-01T20:25:00,740 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,740 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-01T20:25:00,740 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-01T20:25:00,740 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,741 INFO [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733084681459 highest: 1733084681459 on 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,741 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.meta:.meta(num 1733084681422) roll requested 2024-12-01T20:25:00,758 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084700741.meta, exclude list is [], retry=0 2024-12-01T20:25:00,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:00,763 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084700741.meta 2024-12-01T20:25:00,763 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:00,763 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta is not closed yet, will try archiving it next time 2024-12-01T20:25:00,763 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1:(num 1733084681442) roll requested 2024-12-01T20:25:00,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741857_1033 (size=93) 2024-12-01T20:25:00,764 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta not finished, retry = 0 2024-12-01T20:25:00,775 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084700763, exclude list is [], retry=0 2024-12-01T20:25:00,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:00,778 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084700763 2024-12-01T20:25:00,779 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:00,779 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 is not closed yet, will try archiving it next time 2024-12-01T20:25:00,779 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0:(num 1733084681459) roll requested 2024-12-01T20:25:00,779 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084667472 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084667472 2024-12-01T20:25:00,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741858_1034 (size=30490) 2024-12-01T20:25:00,780 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 not finished, retry = 0 2024-12-01T20:25:00,791 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084700779, exclude list is [], retry=0 2024-12-01T20:25:00,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:00,795 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084700779 2024-12-01T20:25:00,795 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:00,795 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 is not closed yet, will try archiving it next time 2024-12-01T20:25:00,796 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 not finished, retry = 0 2024-12-01T20:25:00,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741859_1035 (size=1198) 2024-12-01T20:25:00,801 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-01T20:25:00,801 INFO [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733084700779 on 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:00,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:00,819 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-01T20:25:00,819 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '9168bcbe98d9,38993,1733084657496' in zk 2024-12-01T20:25:00,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,823 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-01T20:25:00,823 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-01T20:25:00,823 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-01T20:25:00,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:00,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:00,824 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:00,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:00,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:00,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:00,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:00,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '9168bcbe98d9,38993,1733084657496': 2024-12-01T20:25:00,827 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-01T20:25:00,827 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '9168bcbe98d9,38993,1733084657496' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-01T20:25:00,827 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-01T20:25:00,827 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-01T20:25:00,827 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-01T20:25:00,827 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-01T20:25:00,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-01T20:25:00,834 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,834 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,834 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:00,834 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,834 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:00,835 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:00,835 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-01T20:25:00,835 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:25:00,835 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,835 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,835 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:00,836 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:00,836 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:00,836 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,837 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,837 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:00,837 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:00,837 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-01T20:25:00,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:00,845 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-01T20:25:00,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:00,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:25:00,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-01T20:25:00,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:00,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:00,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:00,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:00,845 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-01T20:25:00,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-01T20:25:00,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-01T20:25:00,845 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2024-12-01T20:25:00,845 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-01T20:25:00,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:25:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-01T20:25:00,860 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {9168bcbe98d9:38993=1733084667472} newestTimestamps: {9168bcbe98d9:38993=1733084681459} 2024-12-01T20:25:00,863 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 9168bcbe98d9%2C38993%2C1733084657496.meta.1733084661072.meta 2024-12-01T20:25:00,863 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta 2024-12-01T20:25:00,863 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 9168bcbe98d9%2C38993%2C1733084657496.meta.1733084700741.meta 2024-12-01T20:25:00,863 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 2024-12-01T20:25:00,863 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084700779 2024-12-01T20:25:00,863 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 2024-12-01T20:25:00,863 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084700763 2024-12-01T20:25:00,866 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta 2024-12-01T20:25:00,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741891_1067 (size=364) 2024-12-01T20:25:01,279 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17330846657681 target: hdfs://localhost:44065/backupUT/backup_1733084696327/ns2/test-17330846657681 descriptor: 'ns2:test-17330846657681', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:25:01,279 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-01T20:25:01,284 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17330846657681 2024-12-01T20:25:01,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741892_1068 (size=50) 2024-12-01T20:25:01,394 DEBUG [master/9168bcbe98d9:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733084665768 2024-12-01T20:25:01,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741893_1069 (size=950) 2024-12-01T20:25:01,717 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17330846657681 2024-12-01T20:25:01,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741894_1070 (size=950) 2024-12-01T20:25:01,809 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/.tabledesc/.tableinfo.0000000002.908 2024-12-01T20:25:01,809 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/.tabledesc/.tableinfo.0000000003.950 2024-12-01T20:25:01,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733084665768 in 420 msec 2024-12-01T20:25:02,138 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1733084665768 target: hdfs://localhost:44065/backupUT/backup_1733084696327/default/test-1733084665768 descriptor: 'test-1733084665768', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:25:02,138 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-01T20:25:02,145 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1733084665768 2024-12-01T20:25:02,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741895_1071 (size=53) 2024-12-01T20:25:02,553 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1733084665768 2024-12-01T20:25:02,565 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:44065/backupUT/.tmp/backup_1733084696327 from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459;hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 2024-12-01T20:25:02,566 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63f88255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:02,566 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:02,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:02,567 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:02,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:02,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:02,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfdfa7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:02,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:02,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:02,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:02,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:02,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bac4e58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:02,570 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T20:25:02,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 9168bcbe98d9,39089,1733084655909 2024-12-01T20:25:02,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c792097 2024-12-01T20:25:02,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T20:25:02,572 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T20:25:02,577 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-01T20:25:02,577 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17330846657681 2024-12-01T20:25:02,577 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17330846657681,, stopping at row=ns2:test-17330846657681 ,, for max=2147483647 with caching=100 2024-12-01T20:25:02,578 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:02,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:02,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:02,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:02,582 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17330846657681: ns2:test-17330846657681; 2024-12-01T20:25:02,582 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1733084665768 2024-12-01T20:25:02,583 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733084665768,, stopping at row=test-1733084665768 ,, for max=2147483647 with caching=100 2024-12-01T20:25:02,585 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1733084665768: test-1733084665768; 2024-12-01T20:25:02,585 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-01T20:25:02,585 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/hbase-staging/partitions_999e27b5-ba7a-42ca-8415-a132e164a4f8 2024-12-01T20:25:02,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741896_1072 (size=192) 2024-12-01T20:25:03,013 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,014 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,015 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-15144840754145583271.jar 2024-12-01T20:25:03,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:03,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:25:03,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:25:03,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:25:03,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:25:03,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:25:03,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:25:03,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:25:03,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:25:03,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:25:03,802 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:25:03,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:03,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:03,804 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17330846657681,default:test-1733084665768 2024-12-01T20:25:03,804 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:25:03,805 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:25:03,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:03,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:03,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:03,805 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:04,583 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-11465807794081740108.jar 2024-12-01T20:25:04,594 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:25:04,619 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 for WAL files 2024-12-01T20:25:04,620 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 2024-12-01T20:25:04,620 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 for WAL files 2024-12-01T20:25:04,621 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 2024-12-01T20:25:04,711 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local1172997462_0003_24654ebc-1ef1-430b-97ca-deeea8cff44e/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-01T20:25:04,711 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local1172997462_0003_24654ebc-1ef1-430b-97ca-deeea8cff44e/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-01T20:25:04,798 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 for hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 (-9223372036854775808:9223372036854775807) length:30490 2024-12-01T20:25:04,820 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-01T20:25:04,821 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-01T20:25:04,857 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 for hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 (-9223372036854775808:9223372036854775807) length:1198 2024-12-01T20:25:04,861 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-01T20:25:04,861 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-01T20:25:04,938 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ce0553, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:04,938 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:04,938 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:04,939 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:04,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:04,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:04,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469f4791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:04,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:04,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:04,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:04,941 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:04,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49941835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:04,943 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:04,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:04,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:04,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:04,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17330846657681', row='row-t20', locateType=CURRENT is [region=ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a., hostname=9168bcbe98d9,38993,1733084657496, seqNum=2] 2024-12-01T20:25:04,948 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-01T20:25:04,948 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:25:04,948 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:04,948 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:04,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:04,948 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-12-01T20:25:04,948 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 9168bcbe98d9 2024-12-01T20:25:04,953 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/backupUT/.tmp/backup_1733084696327/_temporary/0/_temporary/attempt_local1172997462_0003_r_000000_0/ns2/test-17330846657681/f/ac18fdeba7ba430dbd037403344f0ba6 is 38, key is row-t20/f:q1/1733084694973/Put/seqid=0 2024-12-01T20:25:04,955 WARN [Thread-949 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [9168bcbe98d9:38993] Specified favored nodes: [9168bcbe98d9:38993] 2024-12-01T20:25:04,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741897_1073 (size=5275) 2024-12-01T20:25:05,394 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733f9409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:05,394 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:05,394 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:05,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:05,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:05,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:05,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d5bba97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:05,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:05,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:05,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:05,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:05,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4eb8dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:05,397 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:05,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:05,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:05,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:05,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733084665768', row='row-test-1733084665768-1-0', locateType=CURRENT is [region=test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., hostname=9168bcbe98d9,38993,1733084657496, seqNum=107] 2024-12-01T20:25:05,400 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-01T20:25:05,401 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:25:05,401 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:05,401 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:05,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:05,401 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733084665768-1-0] 2024-12-01T20:25:05,401 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 9168bcbe98d9 2024-12-01T20:25:05,415 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea9e14d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:05,415 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:05,415 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:05,416 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:05,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:05,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:05,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a27887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:05,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:05,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:05,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:05,417 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:05,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79885eb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:05,418 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:05,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:05,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:05,420 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:05,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733084665768', row='row-test-1733084665768-3-0', locateType=CURRENT is [region=test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., hostname=9168bcbe98d9,38993,1733084657496, seqNum=107] 2024-12-01T20:25:05,421 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-01T20:25:05,421 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:25:05,421 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:05,421 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:05,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:05,422 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733084665768-3-0] 2024-12-01T20:25:05,422 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: 9168bcbe98d9 2024-12-01T20:25:05,433 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/backupUT/.tmp/backup_1733084696327/_temporary/0/_temporary/attempt_local1172997462_0003_r_000001_0/default/test-1733084665768/f/0e6f498e3def4a39aee77683e1b726cc is 59, key is row-test-1733084665768-1-10/f:q1/1733084694607/Put/seqid=0 2024-12-01T20:25:05,435 WARN [Thread-961 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [9168bcbe98d9:38993] Specified favored nodes: [9168bcbe98d9:38993] 2024-12-01T20:25:05,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741898_1074 (size=10770) 2024-12-01T20:25:05,839 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/backupUT/.tmp/backup_1733084696327/_temporary/0/_temporary/attempt_local1172997462_0003_r_000001_0/default/test-1733084665768/mob/2bfe97b082bb421881c5c20c7fcb9aa4 is 63, key is row-test-1733084665768-3-100/mob:q1/1733084694933/Put/seqid=0 2024-12-01T20:25:05,844 WARN [Thread-963 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [9168bcbe98d9:38993] Specified favored nodes: [9168bcbe98d9:38993] 2024-12-01T20:25:05,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741899_1075 (size=11811) 2024-12-01T20:25:06,760 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:44065/backupUT 2024-12-01T20:25:06,760 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1733084696327 2024-12-01T20:25:06,760 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-12-01T20:25:06,776 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:44065/backupUT/.tmp/backup_1733084696327, hdfs://localhost:44065/backupUT] 2024-12-01T20:25:06,844 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:25:06,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741900_1076 (size=5275) 2024-12-01T20:25:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741901_1077 (size=10770) 2024-12-01T20:25:07,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741902_1078 (size=11811) 2024-12-01T20:25:07,943 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-12-01T20:25:07,946 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1733084696327 2024-12-01T20:25:07,947 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27856 bytes copied." 2024-12-01T20:25:07,947 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1639986097_0004 completed: true true 2024-12-01T20:25:07,948 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3825262 FILE: Number of bytes written=6215161 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231467 HDFS: Number of bytes written=2433185 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=249 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27856 Bytes Copied=27856 Bytes Expected=27856 Files Copied=4 DIR_COPY=8 2024-12-01T20:25:07,948 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:44065/backupUT/.tmp/backup_1733084696327 to hdfs://localhost:44065/backupUT finished. 2024-12-01T20:25:07,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:ns2:test-17330846657681:', locateType=CURRENT is [region=backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72., hostname=9168bcbe98d9,38993,1733084657496, seqNum=2] 2024-12-01T20:25:07,970 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733084696327 2024-12-01T20:25:07,974 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733084675563} 2024-12-01T20:25:07,974 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-01T20:25:07,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741903_1079 (size=384) 2024-12-01T20:25:08,382 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:44065/backupUT/backup_1733084696327/.backup.manifest 2024-12-01T20:25:08,382 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733084696327 finished: type=INCREMENTAL,tablelist=ns2:test-17330846657681;test-1733084665768,targetRootDir=hdfs://localhost:44065/backupUT,startts=1733084700608,completets=1733084707970,bytescopied=0 2024-12-01T20:25:08,382 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-01T20:25:08,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-01T20:25:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-01T20:25:08,402 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-01T20:25:08,406 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-01T20:25:08,410 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733084696327 completed. 2024-12-01T20:25:08,410 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-12-01T20:25:08,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:08,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:08,427 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:44065/backupUT/backup_1733084696327/.backup.manifest 2024-12-01T20:25:08,429 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1733084696327/.backup.manifest 2024-12-01T20:25:08,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table test-1733084665768 from 'test-1733084665768', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1733084665768', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:25:08,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1733084665768 2024-12-01T20:25:08,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741904_1080 (size=908) 2024-12-01T20:25:08,846 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/.tabledesc/.tableinfo.0000000003.950 2024-12-01T20:25:08,846 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/.tabledesc/.tableinfo.0000000004.908 2024-12-01T20:25:08,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1733084665768}] 2024-12-01T20:25:08,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE}] 2024-12-01T20:25:08,857 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE 2024-12-01T20:25:08,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=CLOSING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:08,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE because future has completed 2024-12-01T20:25:08,861 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T20:25:08,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:25:09,014 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,015 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T20:25:09,015 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 00ff3d9048bd8163f6ce55a5b39c9f69, disabling compactions & flushes 2024-12-01T20:25:09,015 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:09,015 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:09,015 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. after waiting 0 ms 2024-12-01T20:25:09,016 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:09,027 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-12-01T20:25:09,028 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:25:09,028 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:09,028 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 00ff3d9048bd8163f6ce55a5b39c9f69: Waiting for close lock at 1733084709015Running coprocessor pre-close hooks at 1733084709015Disabling compacts and flushes for region at 1733084709015Disabling writes for close at 1733084709016 (+1 ms)Writing region close event to WAL at 1733084709020 (+4 ms)Running coprocessor post-close hooks at 1733084709028 (+8 ms)Closed at 1733084709028 2024-12-01T20:25:09,028 WARN [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 00ff3d9048bd8163f6ce55a5b39c9f69 to self. 2024-12-01T20:25:09,031 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,031 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=CLOSED 2024-12-01T20:25:09,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:25:09,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-01T20:25:09,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 in 173 msec 2024-12-01T20:25:09,037 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE; state=CLOSED, location=9168bcbe98d9,38993,1733084657496; forceNewPlan=false, retain=true 2024-12-01T20:25:09,189 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:09,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE because future has completed 2024-12-01T20:25:09,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:25:09,355 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:09,356 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 00ff3d9048bd8163f6ce55a5b39c9f69, NAME => 'test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:25:09,357 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:25:09,357 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733084665768 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,357 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:25:09,358 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,358 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,361 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,362 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ff3d9048bd8163f6ce55a5b39c9f69 columnFamilyName f 2024-12-01T20:25:09,363 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:09,373 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/7d5ed29d1a5a4760bc41d5b043afaf11 2024-12-01T20:25:09,379 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 2024-12-01T20:25:09,379 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(327): Store=00ff3d9048bd8163f6ce55a5b39c9f69/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:25:09,380 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,381 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ff3d9048bd8163f6ce55a5b39c9f69 columnFamilyName f2 2024-12-01T20:25:09,381 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:09,382 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(327): Store=00ff3d9048bd8163f6ce55a5b39c9f69/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:25:09,382 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,383 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00ff3d9048bd8163f6ce55a5b39c9f69 columnFamilyName mob 2024-12-01T20:25:09,383 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:09,390 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf 2024-12-01T20:25:09,395 DEBUG [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/ec20565cf6cf4889a92097059cb26fc4 2024-12-01T20:25:09,395 INFO [StoreOpener-00ff3d9048bd8163f6ce55a5b39c9f69-1 {}] regionserver.HStore(327): Store=00ff3d9048bd8163f6ce55a5b39c9f69/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:25:09,396 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,397 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,398 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,399 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,399 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,400 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1733084665768 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T20:25:09,402 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,403 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 00ff3d9048bd8163f6ce55a5b39c9f69; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11379472800, jitterRate=0.05979599058628082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T20:25:09,403 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:09,405 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 00ff3d9048bd8163f6ce55a5b39c9f69: Running coprocessor pre-open hook at 1733084709358Writing region info on filesystem at 1733084709358Initializing all the Stores at 1733084709360 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084709360Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084709360Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084709361 (+1 ms)Cleaning up temporary data from old regions at 1733084709399 (+38 ms)Running coprocessor post-open hooks at 1733084709403 (+4 ms)Region opened successfully at 1733084709405 (+2 ms) 2024-12-01T20:25:09,406 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., pid=46, masterSystemTime=1733084709348 2024-12-01T20:25:09,408 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:09,408 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:09,409 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=00ff3d9048bd8163f6ce55a5b39c9f69, regionState=OPEN, openSeqNum=338, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:09,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:25:09,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-12-01T20:25:09,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69, server=9168bcbe98d9,38993,1733084657496 in 217 msec 2024-12-01T20:25:09,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-12-01T20:25:09,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733084665768, region=00ff3d9048bd8163f6ce55a5b39c9f69, REOPEN/MOVE in 557 msec 2024-12-01T20:25:09,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-01T20:25:09,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1733084665768 in 564 msec 2024-12-01T20:25:09,419 DEBUG [PEWorker-2 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1733084665768 2024-12-01T20:25:09,425 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f3/d5889c07403849ceb7069c51feadb85f 2024-12-01T20:25:09,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733084665768 in 995 msec 2024-12-01T20:25:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-01T20:25:09,429 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1733084665768 completed 2024-12-01T20:25:09,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,464 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-01T20:25:09,468 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733084709429 2024-12-01T20:25:09,470 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1733084696327} 2024-12-01T20:25:09,470 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733084675563} 2024-12-01T20:25:09,470 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-12-01T20:25:09,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,510 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1733084665768 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1733084665768 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T20:25:09,513 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1733084709429,startts=0,failedts=1733084709511,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1733084665768 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-12-01T20:25:09,513 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-12-01T20:25:09,515 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-12-01T20:25:09,515 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-01T20:25:09,517 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-12-01T20:25:09,517 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-12-01T20:25:09,518 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-01T20:25:09,520 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1733084709429 failed. 2024-12-01T20:25:09,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:09,539 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1733084665768, ns2:test-17330846657681, ns3:test-17330846657682, ns4:test-17330846657683] 2024-12-01T20:25:09,539 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-01T20:25:09,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:25:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084709544 (current time:1733084709544). 2024-12-01T20:25:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:25:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-01T20:25:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7243f82d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:09,545 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:09,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:09,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:09,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c87ac30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:09,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:09,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:09,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:09,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:09,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c4ca2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:09,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:09,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:09,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:09,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:25:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:25:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:09,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:25:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:25:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-12-01T20:25:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-01T20:25:09,554 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:25:09,555 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:25:09,557 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:25:09,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741905_1081 (size=52) 2024-12-01T20:25:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-01T20:25:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-01T20:25:09,964 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:25:09,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4}] 2024-12-01T20:25:09,966 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:10,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-12-01T20:25:10,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:10,121 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing 8230de2857c5483e5284585a3da8d5f4 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-12-01T20:25:10,127 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/9886096879204cd18c1efaa81b2f7b6b is 158, key is trslm:hdfs://localhost:44065/backupUT\x00ns2:test-17330846657681/meta:log-roll-map/1733084707956/Put/seqid=0 2024-12-01T20:25:10,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741906_1082 (size=5745) 2024-12-01T20:25:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-01T20:25:10,533 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/9886096879204cd18c1efaa81b2f7b6b 2024-12-01T20:25:10,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/0ceeab35d66d40cdb5548bf7581b6332 is 1245, key is session:backup_1733084709429/session:context/1733084709518/Put/seqid=0 2024-12-01T20:25:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741907_1083 (size=6682) 2024-12-01T20:25:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-01T20:25:10,959 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/0ceeab35d66d40cdb5548bf7581b6332 2024-12-01T20:25:10,967 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/9886096879204cd18c1efaa81b2f7b6b as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b 2024-12-01T20:25:10,973 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b, entries=4, sequenceid=31, filesize=5.6 K 2024-12-01T20:25:10,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/0ceeab35d66d40cdb5548bf7581b6332 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332 2024-12-01T20:25:10,980 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332, entries=3, sequenceid=31, filesize=6.5 K 2024-12-01T20:25:10,981 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 8230de2857c5483e5284585a3da8d5f4 in 861ms, sequenceid=31, compaction requested=true 2024-12-01T20:25:10,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for 8230de2857c5483e5284585a3da8d5f4: 2024-12-01T20:25:10,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. for snapshot_backup_system completed. 2024-12-01T20:25:10,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.' region-info for snapshot=snapshot_backup_system 2024-12-01T20:25:10,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:25:10,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b] hfiles 2024-12-01T20:25:10,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc for snapshot=snapshot_backup_system 2024-12-01T20:25:10,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b for snapshot=snapshot_backup_system 2024-12-01T20:25:10,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332] hfiles 2024-12-01T20:25:10,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 for snapshot=snapshot_backup_system 2024-12-01T20:25:10,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e for snapshot=snapshot_backup_system 2024-12-01T20:25:10,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332 for snapshot=snapshot_backup_system 2024-12-01T20:25:10,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741908_1084 (size=251) 2024-12-01T20:25:11,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:11,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-01T20:25:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-12-01T20:25:11,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:11,392 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:11,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-12-01T20:25:11,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 in 1.4300 sec 2024-12-01T20:25:11,398 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:25:11,400 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:25:11,401 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:25:11,401 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-01T20:25:11,402 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-01T20:25:11,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741909_1085 (size=888) 2024-12-01T20:25:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-01T20:25:11,816 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:25:11,824 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:25:11,825 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_backup_system 2024-12-01T20:25:11,826 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:25:11,827 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-12-01T20:25:11,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2750 sec 2024-12-01T20:25:11,840 DEBUG [master/9168bcbe98d9:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733084665768 2024-12-01T20:25:11,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741910_1086 (size=950) 2024-12-01T20:25:12,254 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/.tabledesc/.tableinfo.0000000004.908 2024-12-01T20:25:12,254 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/.tabledesc/.tableinfo.0000000005.950 2024-12-01T20:25:12,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733084665768 in 415 msec 2024-12-01T20:25:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-01T20:25:13,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-01T20:25:13,719 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733084709521 started at 1733084713719. 2024-12-01T20:25:13,725 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733084709521 has been started. 2024-12-01T20:25:13,733 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-12-01T20:25:13,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-01T20:25:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-01T20:25:13,735 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-01T20:25:13,735 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-01T20:25:13,735 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-01T20:25:13,735 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-01T20:25:13,737 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,737 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:13,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-01T20:25:13,802 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,803 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-01T20:25:13,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:25:13,804 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,804 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-01T20:25:13,805 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:13,806 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-01T20:25:13,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:13,808 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-01T20:25:13,808 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-01T20:25:13,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-01T20:25:13,809 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-01T20:25:13,809 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-01T20:25:13,809 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-01T20:25:13,809 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-01T20:25:13,809 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '9168bcbe98d9,38993,1733084657496' joining acquired barrier for procedure (rolllog) in zk 2024-12-01T20:25:13,822 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:13,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:13,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:13,823 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-01T20:25:13,823 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-01T20:25:13,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:13,824 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:13,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:13,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,826 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:13,826 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '9168bcbe98d9,38993,1733084657496' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-01T20:25:13,826 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@9e6a1c9[Count = 0] remaining members to acquire global barrier 2024-12-01T20:25:13,826 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-01T20:25:13,826 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:13,833 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:13,833 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:13,833 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-01T20:25:13,833 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,834 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-01T20:25:13,834 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-01T20:25:13,834 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,834 INFO [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733084700779 highest: 1733084700779 on 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,834 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.meta:.meta(num 1733084700741) roll requested 2024-12-01T20:25:13,853 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084713834.meta, exclude list is [], retry=0 2024-12-01T20:25:13,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:13,857 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084700741.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084713834.meta 2024-12-01T20:25:13,857 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:13,857 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084700741.meta is not closed yet, will try archiving it next time 2024-12-01T20:25:13,857 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1:(num 1733084700763) roll requested 2024-12-01T20:25:13,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741888_1064 (size=1903) 2024-12-01T20:25:13,859 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084700741.meta not finished, retry = 0 2024-12-01T20:25:13,869 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084713857, exclude list is [], retry=0 2024-12-01T20:25:13,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:13,873 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084700763 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084713857 2024-12-01T20:25:13,873 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:13,873 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084700763 is not closed yet, will try archiving it next time 2024-12-01T20:25:13,873 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0:(num 1733084700779) roll requested 2024-12-01T20:25:13,873 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084681442 2024-12-01T20:25:13,874 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084700763 not finished, retry = 0 2024-12-01T20:25:13,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741889_1065 (size=4927) 2024-12-01T20:25:13,884 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873, exclude list is [], retry=0 2024-12-01T20:25:13,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:13,888 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084700779 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 2024-12-01T20:25:13,888 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:13,888 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084700779 is not closed yet, will try archiving it next time 2024-12-01T20:25:13,889 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084700779 not finished, retry = 0 2024-12-01T20:25:13,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741890_1066 (size=93) 2024-12-01T20:25:13,894 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-01T20:25:13,894 INFO [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733084713873 on 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:13,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:13,917 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-01T20:25:13,917 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '9168bcbe98d9,38993,1733084657496' in zk 2024-12-01T20:25:13,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,928 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-01T20:25:13,928 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-01T20:25:13,928 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-01T20:25:13,928 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,928 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:13,928 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:13,928 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:13,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:13,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:13,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:13,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:13,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,931 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '9168bcbe98d9,38993,1733084657496': 2024-12-01T20:25:13,931 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-01T20:25:13,931 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '9168bcbe98d9,38993,1733084657496' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-01T20:25:13,931 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-01T20:25:13,931 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-01T20:25:13,931 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-01T20:25:13,931 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-01T20:25:13,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-01T20:25:13,938 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:13,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:13,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:13,939 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-01T20:25:13,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:25:13,939 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:13,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:13,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:13,940 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,941 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:13,941 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:13,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-01T20:25:13,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:13,949 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-01T20:25:13,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:13,949 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:25:13,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:13,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:13,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:13,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-01T20:25:13,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:13,949 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-01T20:25:13,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-01T20:25:13,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-01T20:25:13,949 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-01T20:25:13,949 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:25:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-01T20:25:13,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-12-01T20:25:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084713960 (current time:1733084713960). 2024-12-01T20:25:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:25:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733084713959_backup_system_bulk VERSION not specified, setting to 2 2024-12-01T20:25:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7569e831, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:13,962 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:13,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:13,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:13,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5d0a69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:13,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:13,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:13,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:13,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:13,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1445d2ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:13,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:13,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:13,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:13,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:25:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:25:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:13,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:25:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-12-01T20:25:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-12-01T20:25:13,971 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:25:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-01T20:25:13,972 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:25:13,975 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:25:13,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741914_1090 (size=76) 2024-12-01T20:25:13,990 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084700779 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084700779 2024-12-01T20:25:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-01T20:25:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-01T20:25:14,384 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:25:14,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c27f3c1374897bd332c0d1fd4656bf72}] 2024-12-01T20:25:14,386 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:25:14,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-12-01T20:25:14,540 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:25:14,541 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for c27f3c1374897bd332c0d1fd4656bf72: 2024-12-01T20:25:14,541 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. for snapshot_1733084713959_backup_system_bulk completed. 2024-12-01T20:25:14,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72.' region-info for snapshot=snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:14,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:25:14,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-01T20:25:14,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-01T20:25:14,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741915_1091 (size=61) 2024-12-01T20:25:14,560 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-01T20:25:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-01T20:25:14,817 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region a3c8df032a6e62b8e874b176ca842b1a, had cached 0 bytes from a total of 8413 2024-12-01T20:25:14,955 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:25:14,955 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-01T20:25:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-01T20:25:14,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733084713959_backup_system_bulk on region c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:25:14,958 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:25:14,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-01T20:25:14,965 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:25:14,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure c27f3c1374897bd332c0d1fd4656bf72 in 576 msec 2024-12-01T20:25:14,967 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:25:14,968 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:25:14,968 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:14,969 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:14,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741916_1092 (size=702) 2024-12-01T20:25:14,992 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T20:25:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-01T20:25:15,385 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:25:15,390 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:25:15,391 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084713959_backup_system_bulk to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:15,392 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:25:15,392 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-12-01T20:25:15,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733084713959_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.4230 sec 2024-12-01T20:25:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-01T20:25:16,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-12-01T20:25:16,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } 2024-12-01T20:25:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084716130 (current time:1733084716130). 2024-12-01T20:25:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:25:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733084716129_ns3_test-17330846657682 VERSION not specified, setting to 2 2024-12-01T20:25:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d335949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:16,135 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:16,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:16,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:16,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3daac3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:16,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:16,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:16,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:16,138 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:16,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f237753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:16,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:16,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:16,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:16,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:25:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:25:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:16,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:25:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } 2024-12-01T20:25:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-12-01T20:25:16,146 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:25:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-01T20:25:16,147 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:25:16,151 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:25:16,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741917_1093 (size=86) 2024-12-01T20:25:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-01T20:25:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-01T20:25:16,614 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:25:16,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure fb2648ec811d6e10defe644f33b327cd}] 2024-12-01T20:25:16,618 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:25:16,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-12-01T20:25:16,773 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:25:16,773 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for fb2648ec811d6e10defe644f33b327cd: 2024-12-01T20:25:16,773 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. for snapshot_1733084716129_ns3_test-17330846657682 completed. 2024-12-01T20:25:16,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd.' region-info for snapshot=snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:16,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:25:16,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-01T20:25:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-01T20:25:16,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741918_1094 (size=52) 2024-12-01T20:25:17,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:25:17,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-01T20:25:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-01T20:25:17,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733084716129_ns3_test-17330846657682 on region fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:25:17,190 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:25:17,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-01T20:25:17,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure fb2648ec811d6e10defe644f33b327cd in 577 msec 2024-12-01T20:25:17,196 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:25:17,197 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:25:17,198 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:25:17,198 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:17,199 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:17,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741919_1095 (size=443) 2024-12-01T20:25:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-01T20:25:17,615 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:25:17,623 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:25:17,623 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084716129_ns3_test-17330846657682 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:17,625 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:25:17,625 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-12-01T20:25:17,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733084716129_ns3_test-17330846657682 table=ns3:test-17330846657682 type=FLUSH ttl=0 } in 1.4810 sec 2024-12-01T20:25:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-01T20:25:18,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17330846657682 completed 2024-12-01T20:25:18,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } 2024-12-01T20:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084718301 (current time:1733084718301). 2024-12-01T20:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:25:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733084718299_ns4_test-17330846657683 VERSION not specified, setting to 2 2024-12-01T20:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a590d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:18,305 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:18,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:18,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:18,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@440ffdab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:18,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:18,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:18,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:18,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:18,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8855dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:18,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:18,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:18,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:18,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:18,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:25:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } 2024-12-01T20:25:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-12-01T20:25:18,318 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:25:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-01T20:25:18,319 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:25:18,322 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:25:18,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741920_1096 (size=86) 2024-12-01T20:25:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-01T20:25:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-01T20:25:18,728 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:25:18,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ea117edd9c2b760fcdfda100b3c4ed5c}] 2024-12-01T20:25:18,729 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:25:18,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-12-01T20:25:18,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:25:18,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for ea117edd9c2b760fcdfda100b3c4ed5c: 2024-12-01T20:25:18,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. for snapshot_1733084718299_ns4_test-17330846657683 completed. 2024-12-01T20:25:18,883 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c.' region-info for snapshot=snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:18,883 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:25:18,883 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-01T20:25:18,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741921_1097 (size=52) 2024-12-01T20:25:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-01T20:25:19,294 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:25:19,294 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-01T20:25:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-01T20:25:19,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733084718299_ns4_test-17330846657683 on region ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:25:19,295 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:25:19,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-01T20:25:19,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure ea117edd9c2b760fcdfda100b3c4ed5c in 568 msec 2024-12-01T20:25:19,297 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:25:19,298 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:25:19,299 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:25:19,299 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:19,299 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:19,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741922_1098 (size=443) 2024-12-01T20:25:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-01T20:25:19,712 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:25:19,720 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:25:19,721 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084718299_ns4_test-17330846657683 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:19,723 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:25:19,723 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-12-01T20:25:19,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733084718299_ns4_test-17330846657683 table=ns4:test-17330846657683 type=FLUSH ttl=0 } in 1.4070 sec 2024-12-01T20:25:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-01T20:25:20,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17330846657683 completed 2024-12-01T20:25:20,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } 2024-12-01T20:25:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084720481 (current time:1733084720481). 2024-12-01T20:25:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:25:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733084720479_ns2_test-17330846657681 VERSION not specified, setting to 2 2024-12-01T20:25:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5717325b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:20,486 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:20,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:20,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:20,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f97871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:20,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:20,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:20,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:20,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:20,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5dc6ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:20,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:20,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:20,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:20,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:25:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:25:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:20,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:25:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } 2024-12-01T20:25:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-12-01T20:25:20,498 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:25:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-01T20:25:20,499 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:25:20,502 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:25:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741923_1099 (size=86) 2024-12-01T20:25:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-01T20:25:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-01T20:25:20,912 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:25:20,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a3c8df032a6e62b8e874b176ca842b1a}] 2024-12-01T20:25:20,916 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:25:21,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-12-01T20:25:21,069 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:25:21,069 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing a3c8df032a6e62b8e874b176ca842b1a 1/1 column families, dataSize=170 B heapSize=816 B 2024-12-01T20:25:21,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/.tmp/f/2306a7736ca446d3a283aef60aebf173 is 38, key is row-t20/f:q1/1733084694973/Put/seqid=0 2024-12-01T20:25:21,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741924_1100 (size=5134) 2024-12-01T20:25:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-01T20:25:21,486 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/.tmp/f/2306a7736ca446d3a283aef60aebf173 2024-12-01T20:25:21,498 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 8230de2857c5483e5284585a3da8d5f4, had cached 0 bytes from a total of 28925 2024-12-01T20:25:21,500 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/.tmp/f/2306a7736ca446d3a283aef60aebf173 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/2306a7736ca446d3a283aef60aebf173 2024-12-01T20:25:21,508 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/2306a7736ca446d3a283aef60aebf173, entries=5, sequenceid=111, filesize=5.0 K 2024-12-01T20:25:21,509 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for a3c8df032a6e62b8e874b176ca842b1a in 440ms, sequenceid=111, compaction requested=false 2024-12-01T20:25:21,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for a3c8df032a6e62b8e874b176ca842b1a: 2024-12-01T20:25:21,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. for snapshot_1733084720479_ns2_test-17330846657681 completed. 2024-12-01T20:25:21,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a.' region-info for snapshot=snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:21,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:25:21,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/2306a7736ca446d3a283aef60aebf173] hfiles 2024-12-01T20:25:21,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e for snapshot=snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:21,509 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/2306a7736ca446d3a283aef60aebf173 for snapshot=snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:21,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741925_1101 (size=130) 2024-12-01T20:25:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-01T20:25:21,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:25:21,954 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-01T20:25:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-01T20:25:21,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733084720479_ns2_test-17330846657681 on region a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:25:21,956 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:25:21,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-01T20:25:21,962 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:25:21,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure a3c8df032a6e62b8e874b176ca842b1a in 1.0470 sec 2024-12-01T20:25:21,962 INFO [master/9168bcbe98d9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-01T20:25:21,962 INFO [master/9168bcbe98d9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-01T20:25:21,963 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:25:21,963 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:25:21,963 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:21,964 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:21,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741926_1102 (size=496) 2024-12-01T20:25:22,378 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:25:22,387 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:25:22,388 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084720479_ns2_test-17330846657681 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:22,389 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:25:22,390 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-12-01T20:25:22,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733084720479_ns2_test-17330846657681 table=ns2:test-17330846657681 type=FLUSH ttl=0 } in 1.8940 sec 2024-12-01T20:25:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-01T20:25:22,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17330846657681 completed 2024-12-01T20:25:22,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } 2024-12-01T20:25:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084722660 (current time:1733084722660). 2024-12-01T20:25:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:25:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733084722659_default_test-1733084665768 VERSION not specified, setting to 2 2024-12-01T20:25:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179d1274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:22,665 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:22,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:22,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:22,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c7c0f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:22,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:22,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:22,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:22,667 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:22,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f662d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:22,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:22,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:22,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:22,672 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:25:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:25:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:22,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:25:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } 2024-12-01T20:25:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-12-01T20:25:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-01T20:25:22,675 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:25:22,677 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:25:22,680 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:25:22,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741927_1103 (size=84) 2024-12-01T20:25:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-01T20:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-01T20:25:23,087 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:25:23,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69}] 2024-12-01T20:25:23,090 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:23,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-12-01T20:25:23,245 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:23,246 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 00ff3d9048bd8163f6ce55a5b39c9f69: 2024-12-01T20:25:23,246 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. for snapshot_1733084722659_default_test-1733084665768 completed. 2024-12-01T20:25:23,246 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69.' region-info for snapshot=snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:23,246 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:25:23,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/7d5ed29d1a5a4760bc41d5b043afaf11] hfiles 2024-12-01T20:25:23,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 for snapshot=snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:23,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/7d5ed29d1a5a4760bc41d5b043afaf11 for snapshot=snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:23,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-01T20:25:23,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/ec20565cf6cf4889a92097059cb26fc4] hfiles 2024-12-01T20:25:23,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf for snapshot=snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:23,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/ec20565cf6cf4889a92097059cb26fc4 for snapshot=snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:23,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741928_1104 (size=224) 2024-12-01T20:25:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-01T20:25:23,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:25:23,661 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-01T20:25:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-01T20:25:23,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733084722659_default_test-1733084665768 on region 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:23,663 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:23,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-01T20:25:23,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 00ff3d9048bd8163f6ce55a5b39c9f69 in 577 msec 2024-12-01T20:25:23,670 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:25:23,672 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:25:23,673 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-01T20:25:23,673 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-01T20:25:23,674 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:23,674 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-12-01T20:25:23,674 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:23,675 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-12-01T20:25:23,675 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:23,677 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/mobdir/data/default/test-1733084665768/7d794a83a2f4a86ecdb78d0f8ec3c014/mob/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69] hfiles 2024-12-01T20:25:23,677 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/mobdir/data/default/test-1733084665768/7d794a83a2f4a86ecdb78d0f8ec3c014/mob/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:23,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741929_1105 (size=168) 2024-12-01T20:25:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-01T20:25:24,089 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:25:24,089 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:24,091 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741930_1106 (size=1347) 2024-12-01T20:25:24,512 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:25:24,521 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:25:24,522 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_1733084722659_default_test-1733084665768 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:24,523 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:25:24,523 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-12-01T20:25:24,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733084722659_default_test-1733084665768 table=test-1733084665768 type=FLUSH ttl=0 } in 1.8500 sec 2024-12-01T20:25:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-01T20:25:24,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1733084665768 completed 2024-12-01T20:25:24,839 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1733084709521 2024-12-01T20:25:24,839 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-12-01T20:25:24,839 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-12-01T20:25:24,839 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733084709521_backup:system_bulk 2024-12-01T20:25:24,839 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733084713959_backup_system_bulk to hdfs://localhost:44065/backupUT/backup_1733084709521/backup/system_bulk/ 2024-12-01T20:25:24,839 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-01T20:25:24,881 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44065, inputRoot=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:25:24,881 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-454178622_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44065/backupUT/backup_1733084709521/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44065/backupUT/backup_1733084709521/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:24,883 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-01T20:25:24,887 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084713959_backup_system_bulk to hdfs://localhost:44065/backupUT/backup_1733084709521/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:24,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741931_1107 (size=702) 2024-12-01T20:25:24,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741932_1108 (size=76) 2024-12-01T20:25:25,302 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:25,303 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:25,304 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,083 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-6280354233029428187.jar 2024-12-01T20:25:26,083 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,084 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,084 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,084 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,084 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,084 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,084 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:26,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:26,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:25:26,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:25:26,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:25:26,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:25:26,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:25:26,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:25:26,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:25:26,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:25:26,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:25:26,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:25:26,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:26,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:26,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:26,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:26,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:26,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:26,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:26,122 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:25:26,144 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733084713959_backup_system_bulk' hfile list 2024-12-01T20:25:26,221 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local1889382052_0005_1a3ceac6-6211-411d-8030-39903af65bb8/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-01T20:25:26,221 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local1889382052_0005_1a3ceac6-6211-411d-8030-39903af65bb8/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-01T20:25:27,253 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-01T20:25:27,255 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-01T20:25:27,264 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:27,264 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733084713959_backup_system_bulk finished. 2024-12-01T20:25:27,264 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733084709521_ns3:test-17330846657682 2024-12-01T20:25:27,264 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733084716129_ns3_test-17330846657682 to hdfs://localhost:44065/backupUT/backup_1733084709521/ns3/test-17330846657682/ 2024-12-01T20:25:27,264 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-01T20:25:27,289 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44065, inputRoot=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:25:27,289 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-454178622_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44065/backupUT/backup_1733084709521/ns3/test-17330846657682, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44065/backupUT/backup_1733084709521/ns3/test-17330846657682/.hbase-snapshot/.tmp/snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:27,291 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-01T20:25:27,295 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084716129_ns3_test-17330846657682 to hdfs://localhost:44065/backupUT/backup_1733084709521/ns3/test-17330846657682/.hbase-snapshot/.tmp/snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:27,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741933_1109 (size=443) 2024-12-01T20:25:27,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741934_1110 (size=86) 2024-12-01T20:25:27,712 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:27,713 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:27,713 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-6102096973591215784.jar 2024-12-01T20:25:28,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:28,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:28,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:25:28,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:25:28,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:25:28,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:25:28,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:25:28,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:25:28,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:25:28,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:25:28,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:25:28,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:25:28,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:28,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:28,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:28,496 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:28,496 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:28,496 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:28,496 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:28,527 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:25:28,551 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733084716129_ns3_test-17330846657682' hfile list 2024-12-01T20:25:28,639 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local919869709_0006_3f43d882-079c-4394-a398-1487b78af1d3/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-01T20:25:28,639 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local919869709_0006_3f43d882-079c-4394-a398-1487b78af1d3/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-01T20:25:29,679 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-01T20:25:29,680 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-01T20:25:29,687 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:29,687 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733084716129_ns3_test-17330846657682 finished. 2024-12-01T20:25:29,687 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733084709521_ns4:test-17330846657683 2024-12-01T20:25:29,687 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733084718299_ns4_test-17330846657683 to hdfs://localhost:44065/backupUT/backup_1733084709521/ns4/test-17330846657683/ 2024-12-01T20:25:29,687 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-01T20:25:29,713 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44065, inputRoot=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:25:29,713 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-454178622_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44065/backupUT/backup_1733084709521/ns4/test-17330846657683, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44065/backupUT/backup_1733084709521/ns4/test-17330846657683/.hbase-snapshot/.tmp/snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:29,715 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-01T20:25:29,719 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084718299_ns4_test-17330846657683 to hdfs://localhost:44065/backupUT/backup_1733084709521/ns4/test-17330846657683/.hbase-snapshot/.tmp/snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:29,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741935_1111 (size=443) 2024-12-01T20:25:29,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741936_1112 (size=86) 2024-12-01T20:25:30,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,908 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-16526063109559081013.jar 2024-12-01T20:25:30,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,909 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,910 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:30,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:30,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:25:30,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:25:30,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:25:30,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:25:30,911 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:25:30,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:25:30,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:25:30,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:25:30,912 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:25:30,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:25:30,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:30,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:30,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:30,913 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:30,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:30,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:30,914 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:30,946 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:25:30,968 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733084718299_ns4_test-17330846657683' hfile list 2024-12-01T20:25:31,042 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local2145699149_0007_801af068-1f0d-4484-aba1-c146c56fc01d/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-01T20:25:31,042 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local2145699149_0007_801af068-1f0d-4484-aba1-c146c56fc01d/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-01T20:25:32,081 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-01T20:25:32,084 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-01T20:25:32,092 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:32,093 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733084718299_ns4_test-17330846657683 finished. 2024-12-01T20:25:32,093 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733084709521_ns2:test-17330846657681 2024-12-01T20:25:32,093 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733084720479_ns2_test-17330846657681 to hdfs://localhost:44065/backupUT/backup_1733084709521/ns2/test-17330846657681/ 2024-12-01T20:25:32,093 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-01T20:25:32,118 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44065, inputRoot=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:25:32,119 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-454178622_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44065/backupUT/backup_1733084709521/ns2/test-17330846657681, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44065/backupUT/backup_1733084709521/ns2/test-17330846657681/.hbase-snapshot/.tmp/snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:32,120 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-01T20:25:32,125 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084720479_ns2_test-17330846657681 to hdfs://localhost:44065/backupUT/backup_1733084709521/ns2/test-17330846657681/.hbase-snapshot/.tmp/snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:32,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741937_1113 (size=496) 2024-12-01T20:25:32,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741938_1114 (size=86) 2024-12-01T20:25:32,541 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:32,541 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:32,542 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,324 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-4593222176248523241.jar 2024-12-01T20:25:33,324 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,324 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,324 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,325 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,325 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,325 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,325 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,325 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,326 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:33,326 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:33,326 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:25:33,326 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:25:33,326 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:25:33,327 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:25:33,327 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:25:33,327 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:25:33,327 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:25:33,327 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:25:33,328 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:25:33,328 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:25:33,328 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:33,328 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:33,328 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:33,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:33,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:33,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:33,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:33,361 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:25:33,383 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733084720479_ns2_test-17330846657681' hfile list 2024-12-01T20:25:33,385 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-12-01T20:25:33,456 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local237625209_0008_bea13264-f4d2-4bfd-9a17-0b8246490082/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-01T20:25:33,456 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local237625209_0008_bea13264-f4d2-4bfd-9a17-0b8246490082/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-01T20:25:33,515 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-01T20:25:33,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741939_1115 (size=8413) 2024-12-01T20:25:33,950 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e output=hdfs://localhost:44065/backupUT/backup_1733084709521/ns2/test-17330846657681/archive/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/c527d074387247ea8c82527b03b9793e 2024-12-01T20:25:33,950 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-12-01T20:25:33,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741940_1116 (size=5134) 2024-12-01T20:25:34,381 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/2306a7736ca446d3a283aef60aebf173 output=hdfs://localhost:44065/backupUT/backup_1733084709521/ns2/test-17330846657681/archive/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/f/2306a7736ca446d3a283aef60aebf173 2024-12-01T20:25:34,382 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.012M/sec 2024-12-01T20:25:34,490 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-01T20:25:34,492 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-01T20:25:34,503 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:34,504 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733084720479_ns2_test-17330846657681 finished. 2024-12-01T20:25:34,504 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733084709521_test-1733084665768 2024-12-01T20:25:34,504 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733084722659_default_test-1733084665768 to hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/ 2024-12-01T20:25:34,504 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-01T20:25:34,528 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44065, inputRoot=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e 2024-12-01T20:25:34,528 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-454178622_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/.hbase-snapshot/.tmp/snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:34,530 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-01T20:25:34,535 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_1733084722659_default_test-1733084665768 to hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/.hbase-snapshot/.tmp/snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:34,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741941_1117 (size=1347) 2024-12-01T20:25:34,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741942_1118 (size=84) 2024-12-01T20:25:34,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:34,951 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:34,952 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,731 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-7477235380107771082.jar 2024-12-01T20:25:35,732 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,732 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,732 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,732 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,733 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,733 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,733 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,733 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,733 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:35,734 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:35,734 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:25:35,734 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:25:35,734 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:25:35,734 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:25:35,735 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:25:35,735 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:25:35,735 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:25:35,735 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:25:35,735 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:25:35,735 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:25:35,736 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:35,736 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:35,736 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:35,736 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:35,737 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-01T20:25:35,737 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:35,737 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:35,768 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:25:35,798 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733084722659_default_test-1733084665768' hfile list 2024-12-01T20:25:35,801 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-12-01T20:25:35,887 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local752695940_0009_8f68c819-8d6c-4bfb-ac65-71e579702f6c/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-01T20:25:35,887 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/mapreduce.cluster.local.dir/job_local752695940_0009_8f68c819-8d6c-4bfb-ac65-71e579702f6c/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-01T20:25:35,950 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-01T20:25:35,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741943_1119 (size=13760) 2024-12-01T20:25:36,386 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/ec20565cf6cf4889a92097059cb26fc4 output=hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/archive/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/ec20565cf6cf4889a92097059cb26fc4 2024-12-01T20:25:36,387 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-12-01T20:25:36,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741944_1120 (size=10796) 2024-12-01T20:25:36,808 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/7d5ed29d1a5a4760bc41d5b043afaf11 output=hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/archive/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/7d5ed29d1a5a4760bc41d5b043afaf11 2024-12-01T20:25:36,808 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.025M/sec 2024-12-01T20:25:36,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741945_1121 (size=8413) 2024-12-01T20:25:37,235 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 output=hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/archive/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f/83cf1f7756204306beca715afde1cb30 2024-12-01T20:25:37,235 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-12-01T20:25:37,251 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/mobdir/data/default/test-1733084665768/7d794a83a2f4a86ecdb78d0f8ec3c014/mob/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69 to hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/archive/data/default/test-1733084665768/7d794a83a2f4a86ecdb78d0f8ec3c014/mob/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69" truncated to max limit (512 characters) 2024-12-01T20:25:37,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741946_1122 (size=5591) 2024-12-01T20:25:37,666 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/mobdir/data/default/test-1733084665768/7d794a83a2f4a86ecdb78d0f8ec3c014/mob/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69 output=hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/archive/data/default/test-1733084665768/7d794a83a2f4a86ecdb78d0f8ec3c014/mob/d41d8cd98f00b204e9800998ecf8427e202412018c91bd7cf7c14d6f8fdee06fc7ae7d4d_00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:25:37,666 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.013M/sec 2024-12-01T20:25:37,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741947_1123 (size=5484) 2024-12-01T20:25:38,102 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf output=hdfs://localhost:44065/backupUT/backup_1733084709521/default/test-1733084665768/archive/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/mob/c1f501fd15fc4f31aff54a45b93dcedf 2024-12-01T20:25:38,103 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-12-01T20:25:38,930 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-01T20:25:38,931 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-01T20:25:38,942 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:38,943 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733084722659_default_test-1733084665768 finished. 2024-12-01T20:25:38,943 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-12-01T20:25:38,943 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17330846657682 2024-12-01T20:25:38,943 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17330846657683 2024-12-01T20:25:38,943 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17330846657681 2024-12-01T20:25:38,943 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1733084665768 2024-12-01T20:25:38,953 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733084709521 2024-12-01T20:25:38,953 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-12-01T20:25:38,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741948_1124 (size=209) 2024-12-01T20:25:39,360 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:44065/backupUT/backup_1733084709521/.backup.manifest 2024-12-01T20:25:39,360 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733084709521 finished: type=FULL,tablelist=backup:system_bulk;ns3:test-17330846657682;ns4:test-17330846657683;ns2:test-17330846657681;test-1733084665768,targetRootDir=hdfs://localhost:44065/backupUT,startts=1733084713719,completets=1733084738952,bytescopied=0 2024-12-01T20:25:39,360 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-12-01T20:25:39,360 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:39,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733084713959_backup_system_bulk" type: DISABLED 2024-12-01T20:25:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733084713959_backup_system_bulk 2024-12-01T20:25:39,370 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733084713959_backup_system_bulk for backup backup_1733084709521 succeeded. 2024-12-01T20:25:39,370 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:39,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733084716129_ns3_test-17330846657682" type: DISABLED 2024-12-01T20:25:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733084716129_ns3_test-17330846657682 2024-12-01T20:25:39,376 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733084716129_ns3_test-17330846657682 for backup backup_1733084709521 succeeded. 2024-12-01T20:25:39,376 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:39,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733084718299_ns4_test-17330846657683" type: DISABLED 2024-12-01T20:25:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733084718299_ns4_test-17330846657683 2024-12-01T20:25:39,381 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733084718299_ns4_test-17330846657683 for backup backup_1733084709521 succeeded. 2024-12-01T20:25:39,381 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:39,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733084720479_ns2_test-17330846657681" type: DISABLED 2024-12-01T20:25:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733084720479_ns2_test-17330846657681 2024-12-01T20:25:39,385 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733084720479_ns2_test-17330846657681 for backup backup_1733084709521 succeeded. 2024-12-01T20:25:39,385 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:39,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_1733084722659_default_test-1733084665768" type: DISABLED 2024-12-01T20:25:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733084722659_default_test-1733084665768 2024-12-01T20:25:39,389 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733084722659_default_test-1733084665768 for backup backup_1733084709521 succeeded. 2024-12-01T20:25:39,390 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-01T20:25:39,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-01T20:25:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-01T20:25:39,397 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-01T20:25:39,399 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-01T20:25:39,405 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733084709521 completed. 2024-12-01T20:25:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,490 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-01T20:25:44,495 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733084744431 2024-12-01T20:25:44,499 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733084709521} 2024-12-01T20:25:44,499 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-01T20:25:44,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:44,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:25:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084744607 (current time:1733084744607). 2024-12-01T20:25:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:25:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-01T20:25:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111f6c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:44,609 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:44,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:44,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:44,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6620f20c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:44,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:44,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:44,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:44,611 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:44,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cabd108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:44,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:44,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:44,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:44,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:25:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:25:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:44,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:25:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:25:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-12-01T20:25:44,617 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:25:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-01T20:25:44,618 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:25:44,621 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:25:44,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741949_1125 (size=52) 2024-12-01T20:25:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-01T20:25:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-01T20:25:44,993 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T20:25:45,028 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-01T20:25:45,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4}] 2024-12-01T20:25:45,031 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:45,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38993 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-12-01T20:25:45,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:45,187 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 8230de2857c5483e5284585a3da8d5f4 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-12-01T20:25:45,196 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/59dc97f5a8df47fc8f6c45b3ae837f7c is 158, key is trslm:hdfs://localhost:44065/backupUT\x00ns2:test-17330846657681/meta:log-roll-map/1733084738947/Put/seqid=0 2024-12-01T20:25:45,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741950_1126 (size=6646) 2024-12-01T20:25:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-01T20:25:45,602 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/59dc97f5a8df47fc8f6c45b3ae837f7c 2024-12-01T20:25:45,618 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/4f3b178e218742a5a9a7986a6e238bb7 is 826, key is session:backup_1733084709521/session:context/1733084739398/Put/seqid=0 2024-12-01T20:25:45,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741951_1127 (size=5928) 2024-12-01T20:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-01T20:25:46,024 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/4f3b178e218742a5a9a7986a6e238bb7 2024-12-01T20:25:46,038 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/59dc97f5a8df47fc8f6c45b3ae837f7c as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c 2024-12-01T20:25:46,045 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c, entries=12, sequenceid=42, filesize=6.5 K 2024-12-01T20:25:46,047 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/4f3b178e218742a5a9a7986a6e238bb7 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7 2024-12-01T20:25:46,052 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7, entries=2, sequenceid=42, filesize=5.8 K 2024-12-01T20:25:46,053 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 8230de2857c5483e5284585a3da8d5f4 in 867ms, sequenceid=42, compaction requested=true 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 8230de2857c5483e5284585a3da8d5f4: 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. for snapshot_backup_system completed. 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.' region-info for snapshot=snapshot_backup_system 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c] hfiles 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc for snapshot=snapshot_backup_system 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b for snapshot=snapshot_backup_system 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c for snapshot=snapshot_backup_system 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7] hfiles 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 for snapshot=snapshot_backup_system 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e for snapshot=snapshot_backup_system 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332 for snapshot=snapshot_backup_system 2024-12-01T20:25:46,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7 for snapshot=snapshot_backup_system 2024-12-01T20:25:46,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741952_1128 (size=330) 2024-12-01T20:25:46,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:46,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-01T20:25:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-01T20:25:46,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:46,464 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:46,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-01T20:25:46,470 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-01T20:25:46,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 8230de2857c5483e5284585a3da8d5f4 in 1.4370 sec 2024-12-01T20:25:46,471 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:25:46,472 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:25:46,472 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-01T20:25:46,474 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-01T20:25:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741953_1129 (size=967) 2024-12-01T20:25:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-01T20:25:46,887 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:25:46,898 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:25:46,899 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/snapshot_backup_system 2024-12-01T20:25:46,900 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:25:46,900 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-12-01T20:25:46,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2850 sec 2024-12-01T20:25:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-01T20:25:48,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-01T20:25:48,788 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733084744431 started at 1733084748788. 2024-12-01T20:25:48,791 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733084744431 has been started. 2024-12-01T20:25:48,800 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, ns2:test-17330846657681, ns3:test-17330846657682, ns4:test-17330846657683, test-1733084665768] 2024-12-01T20:25:48,804 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1733084700779for backupID backup_1733084744431 2024-12-01T20:25:48,804 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-12-01T20:25:48,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.3 procedure request for: rolllog-proc 2024-12-01T20:25:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-01T20:25:48,805 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-01T20:25:48,806 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-01T20:25:48,806 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-01T20:25:48,806 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-01T20:25:48,806 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:48,806 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:48,923 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:48,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-01T20:25:48,923 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-01T20:25:48,923 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:25:48,924 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:48,924 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-01T20:25:48,925 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:48,926 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:48,927 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-01T20:25:48,927 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:48,927 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-01T20:25:48,928 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-01T20:25:48,928 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-01T20:25:48,928 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-01T20:25:48,928 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-01T20:25:48,928 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-01T20:25:48,929 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-01T20:25:48,929 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: '9168bcbe98d9,38993,1733084657496' joining acquired barrier for procedure (rolllog) in zk 2024-12-01T20:25:48,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:48,947 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:48,947 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:48,947 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:48,947 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:48,948 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-01T20:25:48,948 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-01T20:25:48,948 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:48,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:48,951 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:48,953 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:48,954 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:48,955 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: '9168bcbe98d9,38993,1733084657496' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-01T20:25:48,956 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6d705da2[Count = 0] remaining members to acquire global barrier 2024-12-01T20:25:48,956 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-01T20:25:48,956 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:48,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:48,968 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:48,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:48,968 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-01T20:25:48,968 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:48,968 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-01T20:25:48,968 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-01T20:25:48,969 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:48,969 INFO [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733084713873 highest: 1733084713873 on 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:48,969 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.meta:.meta(num 1733084713834) roll requested 2024-12-01T20:25:48,987 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084748969.meta, exclude list is [], retry=0 2024-12-01T20:25:48,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:48,991 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084713834.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084748969.meta 2024-12-01T20:25:48,991 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:48,991 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084713834.meta is not closed yet, will try archiving it next time 2024-12-01T20:25:48,991 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1:(num 1733084713857) roll requested 2024-12-01T20:25:48,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741911_1087 (size=93) 2024-12-01T20:25:48,993 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084713834.meta to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084713834.meta 2024-12-01T20:25:49,003 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084748991, exclude list is [], retry=0 2024-12-01T20:25:49,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:49,006 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084713857 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084748991 2024-12-01T20:25:49,007 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:49,007 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084713857 is not closed yet, will try archiving it next time 2024-12-01T20:25:49,007 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0:(num 1733084713873) roll requested 2024-12-01T20:25:49,007 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084700763 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084700763 2024-12-01T20:25:49,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741912_1088 (size=4710) 2024-12-01T20:25:49,008 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084713857 not finished, retry = 0 2024-12-01T20:25:49,019 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084749007, exclude list is [], retry=0 2024-12-01T20:25:49,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36971,DS-0052a839-d63a-4649-9594-9aab663da30f,DISK] 2024-12-01T20:25:49,022 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084749007 2024-12-01T20:25:49,023 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37927:37927)] 2024-12-01T20:25:49,023 DEBUG [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 is not closed yet, will try archiving it next time 2024-12-01T20:25:49,023 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084681459 2024-12-01T20:25:49,024 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 not finished, retry = 0 2024-12-01T20:25:49,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741913_1089 (size=653) 2024-12-01T20:25:49,029 DEBUG [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-01T20:25:49,029 INFO [rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733084749007 on 9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-01T20:25:49,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:49,047 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-01T20:25:49,048 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member '9168bcbe98d9,38993,1733084657496' in zk 2024-12-01T20:25:49,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,062 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-01T20:25:49,062 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-01T20:25:49,062 DEBUG [member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-01T20:25:49,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:49,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:49,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:49,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:49,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:49,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:49,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:49,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,068 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member '9168bcbe98d9,38993,1733084657496': 2024-12-01T20:25:49,069 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-01T20:25:49,069 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: '9168bcbe98d9,38993,1733084657496' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-01T20:25:49,069 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-01T20:25:49,069 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-01T20:25:49,069 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-01T20:25:49,069 INFO [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-01T20:25:49,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:49,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:49,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-01T20:25:49,083 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:49,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:49,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:49,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-01T20:25:49,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-01T20:25:49,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-01T20:25:49,084 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-01T20:25:49,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:25:49,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:49,085 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-01T20:25:49,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-01T20:25:49,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:49,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-01T20:25:49,089 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-01T20:25:49,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:49,104 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:49,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-01T20:25:49,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-01T20:25:49,104 DEBUG [(9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-01T20:25:49,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-01T20:25:49,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-01T20:25:49,105 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-01T20:25:49,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-01T20:25:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-01T20:25:49,120 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {9168bcbe98d9:38993=1733084700779} newestTimestamps: {9168bcbe98d9:38993=1733084713873} 2024-12-01T20:25:49,121 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 9168bcbe98d9%2C38993%2C1733084657496.meta.1733084661072.meta 2024-12-01T20:25:49,121 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 9168bcbe98d9%2C38993%2C1733084657496.meta.1733084700741.meta 2024-12-01T20:25:49,121 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: 9168bcbe98d9%2C38993%2C1733084657496.meta.1733084748969.meta 2024-12-01T20:25:49,121 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 2024-12-01T20:25:49,121 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084749007 2024-12-01T20:25:49,121 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084713857 2024-12-01T20:25:49,121 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084748991 2024-12-01T20:25:49,122 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084681422.meta 2024-12-01T20:25:49,122 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.meta.1733084713834.meta 2024-12-01T20:25:49,125 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 2024-12-01T20:25:49,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741957_1133 (size=640) 2024-12-01T20:25:49,536 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:44065/backupUT/backup_1733084744431/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:25:49,536 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-01T20:25:49,542 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-12-01T20:25:49,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741958_1134 (size=45) 2024-12-01T20:25:49,948 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-12-01T20:25:49,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741959_1135 (size=390) 2024-12-01T20:25:50,363 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17330846657682 target: hdfs://localhost:44065/backupUT/backup_1733084744431/ns3/test-17330846657682 descriptor: 'ns3:test-17330846657682', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:25:50,364 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-01T20:25:50,366 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17330846657682 2024-12-01T20:25:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741960_1136 (size=50) 2024-12-01T20:25:50,772 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17330846657682 2024-12-01T20:25:50,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741961_1137 (size=390) 2024-12-01T20:25:51,189 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17330846657683 target: hdfs://localhost:44065/backupUT/backup_1733084744431/ns4/test-17330846657683 descriptor: 'ns4:test-17330846657683', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:25:51,190 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-01T20:25:51,195 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17330846657683 2024-12-01T20:25:51,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741962_1138 (size=50) 2024-12-01T20:25:51,602 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17330846657683 2024-12-01T20:25:51,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741963_1139 (size=364) 2024-12-01T20:25:52,020 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17330846657681 target: hdfs://localhost:44065/backupUT/backup_1733084744431/ns2/test-17330846657681 descriptor: 'ns2:test-17330846657681', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:25:52,020 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-01T20:25:52,023 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17330846657681 2024-12-01T20:25:52,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741964_1140 (size=50) 2024-12-01T20:25:52,431 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17330846657681 2024-12-01T20:25:52,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741965_1141 (size=950) 2024-12-01T20:25:52,850 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1733084665768 target: hdfs://localhost:44065/backupUT/backup_1733084744431/default/test-1733084665768 descriptor: 'test-1733084665768', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:25:52,850 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-01T20:25:52,854 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1733084665768 2024-12-01T20:25:52,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741966_1142 (size=53) 2024-12-01T20:25:53,262 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1733084665768 2024-12-01T20:25:53,279 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:44065/backupUT/.tmp/backup_1733084744431 from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873;hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1.1733084713857 2024-12-01T20:25:53,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6e16ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:53,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:53,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:53,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:53,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:53,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:53,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b504363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:53,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:53,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:53,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:53,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:53,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b60947e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:53,283 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T20:25:53,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is 9168bcbe98d9,39089,1733084655909 2024-12-01T20:25:53,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a37c4e0 2024-12-01T20:25:53,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T20:25:53,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T20:25:53,289 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-01T20:25:53,289 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-12-01T20:25:53,289 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-01T20:25:53,289 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:53,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:53,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:53,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:53,293 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-12-01T20:25:53,293 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17330846657681 2024-12-01T20:25:53,293 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17330846657681,, stopping at row=ns2:test-17330846657681 ,, for max=2147483647 with caching=100 2024-12-01T20:25:53,295 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17330846657681: ns2:test-17330846657681; 2024-12-01T20:25:53,295 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17330846657682 2024-12-01T20:25:53,295 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17330846657682,, stopping at row=ns3:test-17330846657682 ,, for max=2147483647 with caching=100 2024-12-01T20:25:53,297 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17330846657682: ns3:test-17330846657682; 2024-12-01T20:25:53,297 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17330846657683 2024-12-01T20:25:53,298 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17330846657683,, stopping at row=ns4:test-17330846657683 ,, for max=2147483647 with caching=100 2024-12-01T20:25:53,300 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17330846657683: ns4:test-17330846657683; 2024-12-01T20:25:53,300 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1733084665768 2024-12-01T20:25:53,300 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733084665768,, stopping at row=test-1733084665768 ,, for max=2147483647 with caching=100 2024-12-01T20:25:53,302 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1733084665768: test-1733084665768; 2024-12-01T20:25:53,302 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-12-01T20:25:53,302 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/hbase-staging/partitions_7e0aacd6-5a68-42fe-8265-e30b1bc12ea6 2024-12-01T20:25:53,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741967_1143 (size=324) 2024-12-01T20:25:53,715 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:53,716 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:53,717 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,357 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 00ff3d9048bd8163f6ce55a5b39c9f69, had cached 0 bytes from a total of 38453 2024-12-01T20:25:54,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-5769412472833502889.jar 2024-12-01T20:25:54,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,489 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,490 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:54,491 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-01T20:25:54,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-01T20:25:54,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-01T20:25:54,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-01T20:25:54,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-01T20:25:54,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-01T20:25:54,492 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-01T20:25:54,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-01T20:25:54,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-01T20:25:54,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-01T20:25:54,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,493 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-01T20:25:54,494 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-01T20:25:54,494 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,ns2:test-17330846657681,ns3:test-17330846657682,ns4:test-17330846657683,default:test-1733084665768 2024-12-01T20:25:54,494 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:25:54,494 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:25:54,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:54,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:54,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:54,495 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-01T20:25:55,269 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop-8993132530102560870.jar 2024-12-01T20:25:55,279 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-01T20:25:55,303 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 for WAL files 2024-12-01T20:25:55,304 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1733084744431File hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 does not exist. java.io.FileNotFoundException: File hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T20:25:55,305 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1733084744431,startts=1733084748788,failedts=1733084755305,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084713873 does not exist. 2024-12-01T20:25:55,305 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-12-01T20:25:55,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable backup:system 2024-12-01T20:25:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-12-01T20:25:55,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084755321"}]},"ts":"1733084755321"} 2024-12-01T20:25:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-01T20:25:55,323 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-12-01T20:25:55,323 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-12-01T20:25:55,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-12-01T20:25:55,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, UNASSIGN}] 2024-12-01T20:25:55,330 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, UNASSIGN 2024-12-01T20:25:55,331 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=8230de2857c5483e5284585a3da8d5f4, regionState=CLOSING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:55,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, UNASSIGN because future has completed 2024-12-01T20:25:55,333 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T20:25:55,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:25:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-01T20:25:55,487 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:55,487 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T20:25:55,487 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 8230de2857c5483e5284585a3da8d5f4, disabling compactions & flushes 2024-12-01T20:25:55,487 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:55,487 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:55,487 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. after waiting 0 ms 2024-12-01T20:25:55,487 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:55,488 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 8230de2857c5483e5284585a3da8d5f4 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-12-01T20:25:55,495 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/6ac99ecb47334f8b81e5b4e6cf46e600 is 103, key is rslogts:hdfs://localhost:44065/backupUT\x009168bcbe98d9:38993/meta:rs-log-ts/1733084749046/Put/seqid=0 2024-12-01T20:25:55,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741968_1144 (size=5338) 2024-12-01T20:25:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-01T20:25:55,902 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/6ac99ecb47334f8b81e5b4e6cf46e600 2024-12-01T20:25:55,916 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/d99912e0f1ef4d92b62d3fe210f333d2 is 303, key is session:backup_1733084744431/session:context/1733084748789/Put/seqid=0 2024-12-01T20:25:55,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741969_1145 (size=5391) 2024-12-01T20:25:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-01T20:25:56,322 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/d99912e0f1ef4d92b62d3fe210f333d2 2024-12-01T20:25:56,331 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/6ac99ecb47334f8b81e5b4e6cf46e600 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/6ac99ecb47334f8b81e5b4e6cf46e600 2024-12-01T20:25:56,338 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/6ac99ecb47334f8b81e5b4e6cf46e600, entries=1, sequenceid=47, filesize=5.2 K 2024-12-01T20:25:56,340 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/d99912e0f1ef4d92b62d3fe210f333d2 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/d99912e0f1ef4d92b62d3fe210f333d2 2024-12-01T20:25:56,346 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/d99912e0f1ef4d92b62d3fe210f333d2, entries=1, sequenceid=47, filesize=5.3 K 2024-12-01T20:25:56,347 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 8230de2857c5483e5284585a3da8d5f4 in 860ms, sequenceid=47, compaction requested=true 2024-12-01T20:25:56,353 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-12-01T20:25:56,354 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:25:56,354 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:56,354 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 8230de2857c5483e5284585a3da8d5f4: Waiting for close lock at 1733084755487Running coprocessor pre-close hooks at 1733084755487Disabling compacts and flushes for region at 1733084755487Disabling writes for close at 1733084755487Obtaining lock to block concurrent updates at 1733084755488 (+1 ms)Preparing flush snapshotting stores in 8230de2857c5483e5284585a3da8d5f4 at 1733084755488Finished memstore snapshotting backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1733084755488Flushing stores of backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. at 1733084755489 (+1 ms)Flushing 8230de2857c5483e5284585a3da8d5f4/meta: creating writer at 1733084755489Flushing 8230de2857c5483e5284585a3da8d5f4/meta: appending metadata at 1733084755494 (+5 ms)Flushing 8230de2857c5483e5284585a3da8d5f4/meta: closing flushed file at 1733084755494Flushing 8230de2857c5483e5284585a3da8d5f4/session: creating writer at 1733084755912 (+418 ms)Flushing 8230de2857c5483e5284585a3da8d5f4/session: appending metadata at 1733084755915 (+3 ms)Flushing 8230de2857c5483e5284585a3da8d5f4/session: closing flushed file at 1733084755916 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@203f543: reopening flushed file at 1733084756329 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@488a50eb: reopening flushed file at 1733084756339 (+10 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 8230de2857c5483e5284585a3da8d5f4 in 860ms, sequenceid=47, compaction requested=true at 1733084756347 (+8 ms)Writing region close event to WAL at 1733084756349 (+2 ms)Running coprocessor post-close hooks at 1733084756353 (+4 ms)Closed at 1733084756354 (+1 ms) 2024-12-01T20:25:56,357 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:56,357 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=8230de2857c5483e5284585a3da8d5f4, regionState=CLOSED 2024-12-01T20:25:56,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:25:56,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-01T20:25:56,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496 in 1.0290 sec 2024-12-01T20:25:56,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-01T20:25:56,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, UNASSIGN in 1.0350 sec 2024-12-01T20:25:56,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-01T20:25:56,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0420 sec 2024-12-01T20:25:56,369 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084756369"}]},"ts":"1733084756369"} 2024-12-01T20:25:56,371 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-12-01T20:25:56,371 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-12-01T20:25:56,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0560 sec 2024-12-01T20:25:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-01T20:25:56,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-12-01T20:25:56,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1733084756469 2024-12-01T20:25:56,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.3 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:25:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733084756470 (current time:1733084756470). 2024-12-01T20:25:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-01T20:25:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1733084756469 VERSION not specified, setting to 2 2024-12-01T20:25:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5e4013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ClusterIdFetcher(90): Going to request 9168bcbe98d9,39089,-1 for getting cluster id 2024-12-01T20:25:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:25:56,472 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d28d033-e930-4ea2-a0ce-57e1357eb371' 2024-12-01T20:25:56,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:25:56,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d28d033-e930-4ea2-a0ce-57e1357eb371" 2024-12-01T20:25:56,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68f3b104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:56,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9168bcbe98d9,39089,-1] 2024-12-01T20:25:56,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:25:56,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:56,474 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:25:56,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36666366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-01T20:25:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:25:56,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9168bcbe98d9,38993,1733084657496, seqNum=-1] 2024-12-01T20:25:56,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:25:56,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:25:56,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089. 2024-12-01T20:25:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-01T20:25:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:56,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-01T20:25:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:25:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-12-01T20:25:56,480 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-01T20:25:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-01T20:25:56,482 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-01T20:25:56,484 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-01T20:25:56,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741970_1146 (size=81) 2024-12-01T20:25:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-01T20:25:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-01T20:25:56,894 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-12-01T20:25:56,896 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-01T20:25:56,897 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-01T20:25:56,897 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:56,901 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/6ac99ecb47334f8b81e5b4e6cf46e600, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b] hfiles 2024-12-01T20:25:56,901 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc 2024-12-01T20:25:56,902 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c 2024-12-01T20:25:56,902 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/6ac99ecb47334f8b81e5b4e6cf46e600 2024-12-01T20:25:56,902 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b 2024-12-01T20:25:56,902 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:56,907 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/d99912e0f1ef4d92b62d3fe210f333d2] hfiles 2024-12-01T20:25:56,907 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 2024-12-01T20:25:56,907 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332 2024-12-01T20:25:56,907 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e 2024-12-01T20:25:56,907 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7 2024-12-01T20:25:56,907 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/d99912e0f1ef4d92b62d3fe210f333d2 2024-12-01T20:25:56,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741971_1147 (size=409) 2024-12-01T20:25:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-01T20:25:57,316 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 8230de2857c5483e5284585a3da8d5f4, NAME => 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-12-01T20:25:57,319 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-01T20:25:57,321 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-01T20:25:57,321 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1733084756469 2024-12-01T20:25:57,324 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1733084756469 2024-12-01T20:25:57,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741972_1148 (size=1046) 2024-12-01T20:25:57,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-01T20:25:57,741 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-01T20:25:57,753 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-01T20:25:57,753 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1733084756469 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1733084756469 2024-12-01T20:25:57,754 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-01T20:25:57,754 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-12-01T20:25:57,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733084756469 table=backup:system type=FLUSH ttl=0 } in 1.2750 sec 2024-12-01T20:25:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-01T20:25:58,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-01T20:25:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1733084744607 type: FLUSH version: 2 ttl: 0 ) 2024-12-01T20:25:58,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-12-01T20:25:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-01T20:25:58,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741973_1149 (size=635) 2024-12-01T20:25:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-01T20:25:58,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-01T20:25:59,064 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-12-01T20:25:59,065 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-12-01T20:25:59,068 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-01T20:25:59,074 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1733084744607 type: FLUSH version: 2 ttl: 0 2024-12-01T20:25:59,075 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system 2024-12-01T20:25:59,079 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-12-01T20:25:59,080 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(229): region to restore: 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,084 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:59,088 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/6ac99ecb47334f8b81e5b4e6cf46e600 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/6ac99ecb47334f8b81e5b4e6cf46e600 2024-12-01T20:25:59,088 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:59,092 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/d99912e0f1ef4d92b62d3fe210f333d2 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/d99912e0f1ef4d92b62d3fe210f333d2 2024-12-01T20:25:59,092 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1733084744607 type: FLUSH version: 2 ttl: 0 2024-12-01T20:25:59,094 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733084759094"}]},"ts":"9223372036854775807"} 2024-12-01T20:25:59,098 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-01T20:25:59,098 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8230de2857c5483e5284585a3da8d5f4, NAME => 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.', STARTKEY => '', ENDKEY => ''}] 2024-12-01T20:25:59,099 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1733084759095"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733084759095"}]},"ts":"1733084759095"} 2024-12-01T20:25:59,100 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:25:59,101 INFO [PEWorker-5 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-12-01T20:25:59,101 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => 8230de2857c5483e5284585a3da8d5f4, NAME => 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.', STARTKEY => '', ENDKEY => ''}] 2024-12-01T20:25:59,101 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-12-01T20:25:59,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1733084744607 type: FLUSH version: 2 ttl: 0 ) in 461 msec 2024-12-01T20:25:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-01T20:25:59,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1733084756469 2024-12-01T20:25:59,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "hbase-failsafe-snapshot_backup_system-1733084756469" type: DISABLED 2024-12-01T20:25:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1733084756469 2024-12-01T20:25:59,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.3 enable backup:system 2024-12-01T20:25:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-12-01T20:25:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-01T20:25:59,297 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-12-01T20:25:59,297 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084759297"}]},"ts":"1733084759297"} 2024-12-01T20:25:59,299 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-12-01T20:25:59,300 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-12-01T20:25:59,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN}] 2024-12-01T20:25:59,301 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN 2024-12-01T20:25:59,301 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-12-01T20:25:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-01T20:25:59,452 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=8230de2857c5483e5284585a3da8d5f4, regionState=OPENING, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:59,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN because future has completed 2024-12-01T20:25:59,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496}] 2024-12-01T20:25:59,615 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:59,616 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => 8230de2857c5483e5284585a3da8d5f4, NAME => 'backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:25:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-01T20:25:59,617 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-01T20:25:59,617 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,617 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:25:59,618 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,618 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,621 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,622 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8230de2857c5483e5284585a3da8d5f4 columnFamilyName meta 2024-12-01T20:25:59,623 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:59,631 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc 2024-12-01T20:25:59,635 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c 2024-12-01T20:25:59,639 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b 2024-12-01T20:25:59,639 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.HStore(327): Store=8230de2857c5483e5284585a3da8d5f4/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:25:59,640 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,641 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8230de2857c5483e5284585a3da8d5f4 columnFamilyName session 2024-12-01T20:25:59,641 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:25:59,648 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 2024-12-01T20:25:59,653 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332 2024-12-01T20:25:59,656 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e 2024-12-01T20:25:59,660 DEBUG [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7 2024-12-01T20:25:59,661 INFO [StoreOpener-8230de2857c5483e5284585a3da8d5f4-1 {}] regionserver.HStore(327): Store=8230de2857c5483e5284585a3da8d5f4/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:25:59,661 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,662 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,663 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,663 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,664 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,664 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T20:25:59,665 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,666 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened 8230de2857c5483e5284585a3da8d5f4; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9970404640, jitterRate=-0.07143370807170868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T20:25:59,666 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:25:59,667 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for 8230de2857c5483e5284585a3da8d5f4: Running coprocessor pre-open hook at 1733084759618Writing region info on filesystem at 1733084759618Initializing all the Stores at 1733084759620 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084759620Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733084759621 (+1 ms)Cleaning up temporary data from old regions at 1733084759664 (+43 ms)Running coprocessor post-open hooks at 1733084759666 (+2 ms)Region opened successfully at 1733084759666 2024-12-01T20:25:59,667 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., pid=70, masterSystemTime=1733084759609 2024-12-01T20:25:59,669 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 8230de2857c5483e5284585a3da8d5f4:meta, priority=-2147483648, current under compaction store size is 1 2024-12-01T20:25:59,670 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-01T20:25:59,670 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 8230de2857c5483e5284585a3da8d5f4:session, priority=-2147483648, current under compaction store size is 2 2024-12-01T20:25:59,670 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-01T20:25:59,670 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-01T20:25:59,671 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-01T20:25:59,673 DEBUG [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:59,673 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-01T20:25:59,673 INFO [RS_OPEN_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:59,673 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-01T20:25:59,673 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=8230de2857c5483e5284585a3da8d5f4, regionState=OPEN, openSeqNum=51, regionLocation=9168bcbe98d9,38993,1733084657496 2024-12-01T20:25:59,675 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.HStore(1541): 8230de2857c5483e5284585a3da8d5f4/meta is initiating minor compaction (all files) 2024-12-01T20:25:59,675 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.HStore(1541): 8230de2857c5483e5284585a3da8d5f4/session is initiating minor compaction (all files) 2024-12-01T20:25:59,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496 because future has completed 2024-12-01T20:25:59,675 INFO [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8230de2857c5483e5284585a3da8d5f4/session in backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:59,675 INFO [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8230de2857c5483e5284585a3da8d5f4/meta in backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:25:59,675 INFO [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c] into tmpdir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp, totalSize=17.9 K 2024-12-01T20:25:59,675 INFO [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7] into tmpdir=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp, totalSize=22.7 K 2024-12-01T20:25:59,676 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf3de11a9904d678102c77d02ef03f8, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733084679082 2024-12-01T20:25:59,676 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56198cfac43b4c6e967ddfbb9ad053cc, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733084681509 2024-12-01T20:25:59,677 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9886096879204cd18c1efaa81b2f7b6b, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733084700818 2024-12-01T20:25:59,677 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] compactions.Compactor(225): Compacting 253b2be5b03848e5b74c1bafdfce328e, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733084694544 2024-12-01T20:25:59,678 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ceeab35d66d40cdb5548bf7581b6332, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733084708404 2024-12-01T20:25:59,678 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59dc97f5a8df47fc8f6c45b3ae837f7c, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733084713916 2024-12-01T20:25:59,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-12-01T20:25:59,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 8230de2857c5483e5284585a3da8d5f4, server=9168bcbe98d9,38993,1733084657496 in 220 msec 2024-12-01T20:25:59,678 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f3b178e218742a5a9a7986a6e238bb7, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733084739398 2024-12-01T20:25:59,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-01T20:25:59,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=8230de2857c5483e5284585a3da8d5f4, ASSIGN in 378 msec 2024-12-01T20:25:59,681 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733084759680"}]},"ts":"1733084759680"} 2024-12-01T20:25:59,682 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-12-01T20:25:59,682 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-12-01T20:25:59,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 392 msec 2024-12-01T20:25:59,702 INFO [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8230de2857c5483e5284585a3da8d5f4#session#compaction#17 average throughput is 2.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-01T20:25:59,702 INFO [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8230de2857c5483e5284585a3da8d5f4#meta#compaction#16 average throughput is 1.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-01T20:25:59,703 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/ea689511aa284a39885f249dd9c8f148 is 158, key is trslm:hdfs://localhost:44065/backupUT\x00ns2:test-17330846657681/meta:log-roll-map/1733084738947/Put/seqid=0 2024-12-01T20:25:59,703 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/7f57e5521bd04da7879360a8f6ebbd9c is 1245, key is session:backup_1733084709429/session:context/1733084709518/Put/seqid=0 2024-12-01T20:25:59,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741974_1150 (size=8073) 2024-12-01T20:25:59,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741975_1151 (size=6749) 2024-12-01T20:25:59,818 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region a3c8df032a6e62b8e874b176ca842b1a, had cached 0 bytes from a total of 13547 2024-12-01T20:25:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-01T20:25:59,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-12-01T20:25:59,928 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-12-01T20:25:59,928 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-01T20:25:59,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.3 delete name: "snapshot_backup_system" type: DISABLED 2024-12-01T20:25:59,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39089 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-01T20:25:59,943 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-01T20:25:59,943 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-12-01T20:25:59,945 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-01T20:25:59,951 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1733084744431 failed. 2024-12-01T20:25:59,952 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:25:59,952 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:25:59,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:59,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:59,952 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:59,969 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=627 (was 611) Potentially hanging thread: PacketResponder: BP-824697111-172.17.0.3-1733084651933:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-824697111-172.17.0.3-1733084651933:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/ea689511aa284a39885f249dd9c8f148 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/7f57e5521bd04da7879360a8f6ebbd9c java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9168bcbe98d9:38993-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1652935462_20 at /127.0.0.1:34490 [Receiving block BP-824697111-172.17.0.3-1733084651933:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/9168bcbe98d9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-824697111-172.17.0.3-1733084651933:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: '9168bcbe98d9,38993,1733084657496' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1652935462_20 at /127.0.0.1:34496 [Receiving block BP-824697111-172.17.0.3-1733084651933:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(9168bcbe98d9,38993,1733084657496)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (9168bcbe98d9,39089,1733084655909)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1652935462_20 at /127.0.0.1:34508 [Receiving block BP-824697111-172.17.0.3-1733084651933:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_885511112_20 at /127.0.0.1:34440 [Waiting for operation #71] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=641 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=82 (was 150), ProcessCount=18 (was 18), AvailableMemoryMB=7496 (was 8118) 2024-12-01T20:25:59,970 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=627 is superior to 500 2024-12-01T20:25:59,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T20:25:59,977 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:25:59,977 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:25:59,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:59,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:25:59,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:25:59,977 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T20:25:59,977 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=27897051, stopped=false 2024-12-01T20:25:59,978 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9168bcbe98d9,39089,1733084655909 2024-12-01T20:26:00,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-12-01T20:26:00,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-12-01T20:26:00,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:26:00,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:26:00,041 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T20:26:00,043 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-01T20:26:00,043 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:26:00,043 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:26:00,044 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(878): Closing user regions 2024-12-01T20:26:00,043 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-01T20:26:00,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:26:00,044 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for 8230de2857c5483e5284585a3da8d5f4 2024-12-01T20:26:00,045 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9168bcbe98d9,38993,1733084657496' ***** 2024-12-01T20:26:00,045 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T20:26:00,046 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8230de2857c5483e5284585a3da8d5f4, disabling compactions & flushes 2024-12-01T20:26:00,046 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for ea117edd9c2b760fcdfda100b3c4ed5c 2024-12-01T20:26:00,046 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for a3c8df032a6e62b8e874b176ca842b1a 2024-12-01T20:26:00,046 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:26:00,046 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for 00ff3d9048bd8163f6ce55a5b39c9f69 2024-12-01T20:26:00,046 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:26:00,047 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for c27f3c1374897bd332c0d1fd4656bf72 2024-12-01T20:26:00,048 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.HBaseServerBase(468): Stop info server 2024-12-01T20:26:00,051 INFO [regionserver/9168bcbe98d9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-01T20:26:00,051 INFO [regionserver/9168bcbe98d9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-01T20:26:00,053 INFO [RS:0;9168bcbe98d9:38993 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@9984275{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-12-01T20:26:00,055 INFO [RS:0;9168bcbe98d9:38993 {}] server.AbstractConnector(383): Stopped ServerConnector@445a8045{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-01T20:26:00,055 INFO [RS:0;9168bcbe98d9:38993 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:26:00,055 INFO [RS:0;9168bcbe98d9:38993 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2df3a35c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-12-01T20:26:00,056 INFO [RS:0;9168bcbe98d9:38993 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3e9615b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,STOPPED} 2024-12-01T20:26:00,057 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T20:26:00,057 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T20:26:00,057 INFO [RS:0;9168bcbe98d9:38993 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 8230de2857c5483e5284585a3da8d5f4, which we are already trying to CLOSE, but not completed yet 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: ea117edd9c2b760fcdfda100b3c4ed5c, which we are already trying to CLOSE, but not completed yet 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: a3c8df032a6e62b8e874b176ca842b1a, which we are already trying to CLOSE, but not completed yet 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 00ff3d9048bd8163f6ce55a5b39c9f69, which we are already trying to CLOSE, but not completed yet 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: fb2648ec811d6e10defe644f33b327cd, which we are already trying to CLOSE, but not completed yet 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: c27f3c1374897bd332c0d1fd4656bf72, which we are already trying to CLOSE, but not completed yet 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(959): stopping server 9168bcbe98d9,38993,1733084657496 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T20:26:00,058 INFO [RS:0;9168bcbe98d9:38993 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9168bcbe98d9:38993. 2024-12-01T20:26:00,058 DEBUG [RS:0;9168bcbe98d9:38993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:26:00,058 DEBUG [RS:0;9168bcbe98d9:38993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:26:00,059 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T20:26:00,060 INFO [regionserver/9168bcbe98d9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T20:26:00,119 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/meta/ea689511aa284a39885f249dd9c8f148 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/ea689511aa284a39885f249dd9c8f148 2024-12-01T20:26:00,119 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/7f57e5521bd04da7879360a8f6ebbd9c as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/7f57e5521bd04da7879360a8f6ebbd9c 2024-12-01T20:26:00,131 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-12-01T20:26:00,131 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-12-01T20:26:00,134 INFO [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8230de2857c5483e5284585a3da8d5f4/meta of 8230de2857c5483e5284585a3da8d5f4 into ea689511aa284a39885f249dd9c8f148(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T20:26:00,134 INFO [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8230de2857c5483e5284585a3da8d5f4/session of 8230de2857c5483e5284585a3da8d5f4 into 7f57e5521bd04da7879360a8f6ebbd9c(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T20:26:00,134 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8230de2857c5483e5284585a3da8d5f4: 2024-12-01T20:26:00,134 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8230de2857c5483e5284585a3da8d5f4: 2024-12-01T20:26:00,134 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:26:00,134 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:26:00,134 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. after waiting 0 ms 2024-12-01T20:26:00,135 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:26:00,135 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8230de2857c5483e5284585a3da8d5f4 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-12-01T20:26:00,137 INFO [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., storeName=8230de2857c5483e5284585a3da8d5f4/session, priority=12, startTime=1733084759670; duration=0sec 2024-12-01T20:26:00,137 INFO [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., storeName=8230de2857c5483e5284585a3da8d5f4/meta, priority=13, startTime=1733084759669; duration=0sec 2024-12-01T20:26:00,137 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-01T20:26:00,137 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-01T20:26:00,137 DEBUG [RS:0;9168bcbe98d9:38993-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8230de2857c5483e5284585a3da8d5f4:meta 2024-12-01T20:26:00,137 DEBUG [RS:0;9168bcbe98d9:38993-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8230de2857c5483e5284585a3da8d5f4:session 2024-12-01T20:26:00,138 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T20:26:00,138 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T20:26:00,138 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T20:26:00,138 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-12-01T20:26:00,138 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1325): Online Regions={8230de2857c5483e5284585a3da8d5f4=backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., ea117edd9c2b760fcdfda100b3c4ed5c=ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c., a3c8df032a6e62b8e874b176ca842b1a=ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a., 00ff3d9048bd8163f6ce55a5b39c9f69=test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., 1588230740=hbase:meta,,1.1588230740, fb2648ec811d6e10defe644f33b327cd=ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd., c27f3c1374897bd332c0d1fd4656bf72=backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72.} 2024-12-01T20:26:00,138 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T20:26:00,138 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T20:26:00,138 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T20:26:00,138 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T20:26:00,138 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T20:26:00,138 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-12-01T20:26:00,138 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1351): Waiting on 00ff3d9048bd8163f6ce55a5b39c9f69, 1588230740, 8230de2857c5483e5284585a3da8d5f4, a3c8df032a6e62b8e874b176ca842b1a, c27f3c1374897bd332c0d1fd4656bf72, ea117edd9c2b760fcdfda100b3c4ed5c, fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:26:00,140 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/96e1b1e2d0a94ecb998a3bcda69d2519 is 554, key is session:backup_1733084744431/session:context/1733084759944/Put/seqid=0 2024-12-01T20:26:00,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741976_1152 (size=5655) 2024-12-01T20:26:00,158 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/info/9a8223fd9559405c990fa33a1f334015 is 159, key is ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a./info:regioninfo/1733084669839/Put/seqid=0 2024-12-01T20:26:00,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741977_1153 (size=14126) 2024-12-01T20:26:00,339 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1351): Waiting on 00ff3d9048bd8163f6ce55a5b39c9f69, 1588230740, 8230de2857c5483e5284585a3da8d5f4, a3c8df032a6e62b8e874b176ca842b1a, c27f3c1374897bd332c0d1fd4656bf72, ea117edd9c2b760fcdfda100b3c4ed5c, fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:26:00,539 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1351): Waiting on 00ff3d9048bd8163f6ce55a5b39c9f69, 1588230740, 8230de2857c5483e5284585a3da8d5f4, a3c8df032a6e62b8e874b176ca842b1a, c27f3c1374897bd332c0d1fd4656bf72, ea117edd9c2b760fcdfda100b3c4ed5c, fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:26:00,546 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/96e1b1e2d0a94ecb998a3bcda69d2519 2024-12-01T20:26:00,559 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/.tmp/session/96e1b1e2d0a94ecb998a3bcda69d2519 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/96e1b1e2d0a94ecb998a3bcda69d2519 2024-12-01T20:26:00,563 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/info/9a8223fd9559405c990fa33a1f334015 2024-12-01T20:26:00,568 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/96e1b1e2d0a94ecb998a3bcda69d2519, entries=2, sequenceid=57, filesize=5.5 K 2024-12-01T20:26:00,569 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 8230de2857c5483e5284585a3da8d5f4 in 434ms, sequenceid=57, compaction requested=false 2024-12-01T20:26:00,570 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c] to archive 2024-12-01T20:26:00,571 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T20:26:00,576 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/56198cfac43b4c6e967ddfbb9ad053cc 2024-12-01T20:26:00,577 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/9886096879204cd18c1efaa81b2f7b6b 2024-12-01T20:26:00,579 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/meta/59dc97f5a8df47fc8f6c45b3ae837f7c 2024-12-01T20:26:00,585 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9168bcbe98d9:39089 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-01T20:26:00,586 WARN [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [56198cfac43b4c6e967ddfbb9ad053cc=5914, 9886096879204cd18c1efaa81b2f7b6b=5745, 59dc97f5a8df47fc8f6c45b3ae837f7c=6646] 2024-12-01T20:26:00,587 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7] to archive 2024-12-01T20:26:00,587 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/ns/7143619f42d24adb8a39ab6286d556d1 is 43, key is default/ns:d/1733084661227/Put/seqid=0 2024-12-01T20:26:00,588 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T20:26:00,590 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0bf3de11a9904d678102c77d02ef03f8 2024-12-01T20:26:00,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741978_1154 (size=5345) 2024-12-01T20:26:00,591 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/253b2be5b03848e5b74c1bafdfce328e 2024-12-01T20:26:00,593 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/0ceeab35d66d40cdb5548bf7581b6332 2024-12-01T20:26:00,594 DEBUG [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7 to hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/archive/data/backup/system/8230de2857c5483e5284585a3da8d5f4/session/4f3b178e218742a5a9a7986a6e238bb7 2024-12-01T20:26:00,594 WARN [StoreCloser-backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [0bf3de11a9904d678102c77d02ef03f8=5049, 253b2be5b03848e5b74c1bafdfce328e=5535, 0ceeab35d66d40cdb5548bf7581b6332=6682, 4f3b178e218742a5a9a7986a6e238bb7=5928] 2024-12-01T20:26:00,597 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system/8230de2857c5483e5284585a3da8d5f4/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-12-01T20:26:00,598 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:26:00,598 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:26:00,598 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8230de2857c5483e5284585a3da8d5f4: Waiting for close lock at 1733084760045Running coprocessor pre-close hooks at 1733084760046 (+1 ms)Disabling compacts and flushes for region at 1733084760046Disabling writes for close at 1733084760135 (+89 ms)Obtaining lock to block concurrent updates at 1733084760135Preparing flush snapshotting stores in 8230de2857c5483e5284585a3da8d5f4 at 1733084760135Finished memstore snapshotting backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1733084760135Flushing stores of backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. at 1733084760136 (+1 ms)Flushing 8230de2857c5483e5284585a3da8d5f4/session: creating writer at 1733084760136Flushing 8230de2857c5483e5284585a3da8d5f4/session: appending metadata at 1733084760139 (+3 ms)Flushing 8230de2857c5483e5284585a3da8d5f4/session: closing flushed file at 1733084760139Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28ac47d: reopening flushed file at 1733084760558 (+419 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 8230de2857c5483e5284585a3da8d5f4 in 434ms, sequenceid=57, compaction requested=false at 1733084760569 (+11 ms)Writing region close event to WAL at 1733084760594 (+25 ms)Running coprocessor post-close hooks at 1733084760598 (+4 ms)Closed at 1733084760598 2024-12-01T20:26:00,598 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1733084675725.8230de2857c5483e5284585a3da8d5f4. 2024-12-01T20:26:00,598 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ea117edd9c2b760fcdfda100b3c4ed5c, disabling compactions & flushes 2024-12-01T20:26:00,598 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:26:00,598 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:26:00,598 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. after waiting 0 ms 2024-12-01T20:26:00,598 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:26:00,602 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns4/test-17330846657683/ea117edd9c2b760fcdfda100b3c4ed5c/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-01T20:26:00,602 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:26:00,602 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:26:00,602 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ea117edd9c2b760fcdfda100b3c4ed5c: Waiting for close lock at 1733084760598Running coprocessor pre-close hooks at 1733084760598Disabling compacts and flushes for region at 1733084760598Disabling writes for close at 1733084760598Writing region close event to WAL at 1733084760599 (+1 ms)Running coprocessor post-close hooks at 1733084760602 (+3 ms)Closed at 1733084760602 2024-12-01T20:26:00,602 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17330846657683,,1733084673068.ea117edd9c2b760fcdfda100b3c4ed5c. 2024-12-01T20:26:00,602 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a3c8df032a6e62b8e874b176ca842b1a, disabling compactions & flushes 2024-12-01T20:26:00,602 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:26:00,602 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:26:00,602 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. after waiting 0 ms 2024-12-01T20:26:00,602 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:26:00,606 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns2/test-17330846657681/a3c8df032a6e62b8e874b176ca842b1a/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-12-01T20:26:00,606 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:26:00,606 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:26:00,606 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a3c8df032a6e62b8e874b176ca842b1a: Waiting for close lock at 1733084760602Running coprocessor pre-close hooks at 1733084760602Disabling compacts and flushes for region at 1733084760602Disabling writes for close at 1733084760602Writing region close event to WAL at 1733084760603 (+1 ms)Running coprocessor post-close hooks at 1733084760606 (+3 ms)Closed at 1733084760606 2024-12-01T20:26:00,607 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17330846657681,,1733084668635.a3c8df032a6e62b8e874b176ca842b1a. 2024-12-01T20:26:00,607 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 00ff3d9048bd8163f6ce55a5b39c9f69, disabling compactions & flushes 2024-12-01T20:26:00,607 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:26:00,607 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:26:00,607 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. after waiting 0 ms 2024-12-01T20:26:00,607 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:26:00,607 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 00ff3d9048bd8163f6ce55a5b39c9f69 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-12-01T20:26:00,620 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f2/8f2335b6a94b42d6915a7756cfd04a4b is 58, key is row-test-1733084665768-2-0/f2:q1/1733084739422/Put/seqid=0 2024-12-01T20:26:00,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741979_1155 (size=5464) 2024-12-01T20:26:00,739 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1351): Waiting on 00ff3d9048bd8163f6ce55a5b39c9f69, 1588230740, c27f3c1374897bd332c0d1fd4656bf72, fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:26:00,940 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1351): Waiting on 00ff3d9048bd8163f6ce55a5b39c9f69, 1588230740, c27f3c1374897bd332c0d1fd4656bf72, fb2648ec811d6e10defe644f33b327cd 2024-12-01T20:26:00,992 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/ns/7143619f42d24adb8a39ab6286d556d1 2024-12-01T20:26:01,024 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/table/bab939535592482c94cdb7aa1116985c is 59, key is ns2:test-17330846657681/table:state/1733084669854/Put/seqid=0 2024-12-01T20:26:01,024 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f2/8f2335b6a94b42d6915a7756cfd04a4b 2024-12-01T20:26:01,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741980_1156 (size=5866) 2024-12-01T20:26:01,030 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/.tmp/f2/8f2335b6a94b42d6915a7756cfd04a4b as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f2/8f2335b6a94b42d6915a7756cfd04a4b 2024-12-01T20:26:01,036 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/f2/8f2335b6a94b42d6915a7756cfd04a4b, entries=7, sequenceid=348, filesize=5.3 K 2024-12-01T20:26:01,036 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 00ff3d9048bd8163f6ce55a5b39c9f69 in 429ms, sequenceid=348, compaction requested=false 2024-12-01T20:26:01,040 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/default/test-1733084665768/00ff3d9048bd8163f6ce55a5b39c9f69/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-12-01T20:26:01,041 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:26:01,041 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:26:01,041 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 00ff3d9048bd8163f6ce55a5b39c9f69: Waiting for close lock at 1733084760607Running coprocessor pre-close hooks at 1733084760607Disabling compacts and flushes for region at 1733084760607Disabling writes for close at 1733084760607Obtaining lock to block concurrent updates at 1733084760607Preparing flush snapshotting stores in 00ff3d9048bd8163f6ce55a5b39c9f69 at 1733084760607Finished memstore snapshotting test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1733084760607Flushing stores of test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. at 1733084760607Flushing 00ff3d9048bd8163f6ce55a5b39c9f69/f2: creating writer at 1733084760608 (+1 ms)Flushing 00ff3d9048bd8163f6ce55a5b39c9f69/f2: appending metadata at 1733084760620 (+12 ms)Flushing 00ff3d9048bd8163f6ce55a5b39c9f69/f2: closing flushed file at 1733084760620Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c213c2d: reopening flushed file at 1733084761030 (+410 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 00ff3d9048bd8163f6ce55a5b39c9f69 in 429ms, sequenceid=348, compaction requested=false at 1733084761036 (+6 ms)Writing region close event to WAL at 1733084761037 (+1 ms)Running coprocessor post-close hooks at 1733084761041 (+4 ms)Closed at 1733084761041 2024-12-01T20:26:01,041 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1733084665768,,1733084666262.00ff3d9048bd8163f6ce55a5b39c9f69. 2024-12-01T20:26:01,041 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb2648ec811d6e10defe644f33b327cd, disabling compactions & flushes 2024-12-01T20:26:01,041 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:26:01,041 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:26:01,041 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. after waiting 0 ms 2024-12-01T20:26:01,041 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:26:01,044 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/ns3/test-17330846657682/fb2648ec811d6e10defe644f33b327cd/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-01T20:26:01,045 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:26:01,045 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:26:01,045 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb2648ec811d6e10defe644f33b327cd: Waiting for close lock at 1733084761041Running coprocessor pre-close hooks at 1733084761041Disabling compacts and flushes for region at 1733084761041Disabling writes for close at 1733084761041Writing region close event to WAL at 1733084761041Running coprocessor post-close hooks at 1733084761045 (+4 ms)Closed at 1733084761045 2024-12-01T20:26:01,045 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17330846657682,,1733084670910.fb2648ec811d6e10defe644f33b327cd. 2024-12-01T20:26:01,045 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c27f3c1374897bd332c0d1fd4656bf72, disabling compactions & flushes 2024-12-01T20:26:01,045 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:26:01,045 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:26:01,045 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. after waiting 0 ms 2024-12-01T20:26:01,045 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:26:01,048 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/backup/system_bulk/c27f3c1374897bd332c0d1fd4656bf72/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-01T20:26:01,049 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:26:01,049 INFO [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:26:01,049 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c27f3c1374897bd332c0d1fd4656bf72: Waiting for close lock at 1733084761045Running coprocessor pre-close hooks at 1733084761045Disabling compacts and flushes for region at 1733084761045Disabling writes for close at 1733084761045Writing region close event to WAL at 1733084761045Running coprocessor post-close hooks at 1733084761049 (+4 ms)Closed at 1733084761049 2024-12-01T20:26:01,049 DEBUG [RS_CLOSE_REGION-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1733084676884.c27f3c1374897bd332c0d1fd4656bf72. 2024-12-01T20:26:01,140 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-01T20:26:01,140 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-01T20:26:01,140 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-01T20:26:01,340 DEBUG [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-01T20:26:01,429 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/table/bab939535592482c94cdb7aa1116985c 2024-12-01T20:26:01,437 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/info/9a8223fd9559405c990fa33a1f334015 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/info/9a8223fd9559405c990fa33a1f334015 2024-12-01T20:26:01,444 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/info/9a8223fd9559405c990fa33a1f334015, entries=69, sequenceid=59, filesize=13.8 K 2024-12-01T20:26:01,446 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/ns/7143619f42d24adb8a39ab6286d556d1 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/ns/7143619f42d24adb8a39ab6286d556d1 2024-12-01T20:26:01,453 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/ns/7143619f42d24adb8a39ab6286d556d1, entries=7, sequenceid=59, filesize=5.2 K 2024-12-01T20:26:01,454 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/.tmp/table/bab939535592482c94cdb7aa1116985c as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/table/bab939535592482c94cdb7aa1116985c 2024-12-01T20:26:01,460 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/table/bab939535592482c94cdb7aa1116985c, entries=13, sequenceid=59, filesize=5.7 K 2024-12-01T20:26:01,461 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1323ms, sequenceid=59, compaction requested=false 2024-12-01T20:26:01,466 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-12-01T20:26:01,466 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-01T20:26:01,466 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T20:26:01,467 INFO [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T20:26:01,467 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733084760138Running coprocessor pre-close hooks at 1733084760138Disabling compacts and flushes for region at 1733084760138Disabling writes for close at 1733084760138Obtaining lock to block concurrent updates at 1733084760138Preparing flush snapshotting stores in 1588230740 at 1733084760138Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1733084760139 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733084760139Flushing 1588230740/info: creating writer at 1733084760139Flushing 1588230740/info: appending metadata at 1733084760158 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733084760158Flushing 1588230740/ns: creating writer at 1733084760570 (+412 ms)Flushing 1588230740/ns: appending metadata at 1733084760586 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733084760586Flushing 1588230740/table: creating writer at 1733084761005 (+419 ms)Flushing 1588230740/table: appending metadata at 1733084761024 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733084761024Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14f66d63: reopening flushed file at 1733084761436 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f5ce1bc: reopening flushed file at 1733084761445 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77d4140c: reopening flushed file at 1733084761453 (+8 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1323ms, sequenceid=59, compaction requested=false at 1733084761461 (+8 ms)Writing region close event to WAL at 1733084761463 (+2 ms)Running coprocessor post-close hooks at 1733084761466 (+3 ms)Closed at 1733084761467 (+1 ms) 2024-12-01T20:26:01,467 DEBUG [RS_CLOSE_META-regionserver/9168bcbe98d9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T20:26:01,541 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(976): stopping server 9168bcbe98d9,38993,1733084657496; all regions closed. 2024-12-01T20:26:01,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741954_1130 (size=3260) 2024-12-01T20:26:01,554 DEBUG [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs 2024-12-01T20:26:01,555 INFO [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.meta:.meta(num 1733084748969) 2024-12-01T20:26:01,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741956_1132 (size=1037) 2024-12-01T20:26:01,557 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/WALs/9168bcbe98d9,38993,1733084657496/9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0.1733084749007 not finished, retry = 0 2024-12-01T20:26:01,665 DEBUG [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs 2024-12-01T20:26:01,665 INFO [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-0:(num 1733084749007) 2024-12-01T20:26:01,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741955_1131 (size=5275) 2024-12-01T20:26:01,674 DEBUG [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/oldWALs 2024-12-01T20:26:01,674 INFO [RS:0;9168bcbe98d9:38993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9168bcbe98d9%2C38993%2C1733084657496.9168bcbe98d9%2C38993%2C1733084657496.regiongroup-1:(num 1733084748991) 2024-12-01T20:26:01,674 DEBUG [RS:0;9168bcbe98d9:38993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:26:01,674 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T20:26:01,674 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T20:26:01,674 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.ChoreService(370): Chore service for: regionserver/9168bcbe98d9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T20:26:01,674 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T20:26:01,674 INFO [regionserver/9168bcbe98d9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T20:26:01,675 INFO [RS:0;9168bcbe98d9:38993 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38993 2024-12-01T20:26:01,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/9168bcbe98d9,38993,1733084657496 2024-12-01T20:26:01,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-12-01T20:26:01,725 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T20:26:01,736 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9168bcbe98d9,38993,1733084657496] 2024-12-01T20:26:01,745 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/9168bcbe98d9,38993,1733084657496 already deleted, retry=false 2024-12-01T20:26:01,746 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9168bcbe98d9,38993,1733084657496 expired; onlineServers=0 2024-12-01T20:26:01,746 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9168bcbe98d9,39089,1733084655909' ***** 2024-12-01T20:26:01,746 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T20:26:01,746 INFO [M:0;9168bcbe98d9:39089 {}] hbase.HBaseServerBase(468): Stop info server 2024-12-01T20:26:01,751 INFO [M:0;9168bcbe98d9:39089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5c685ba8{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-12-01T20:26:01,752 INFO [M:0;9168bcbe98d9:39089 {}] server.AbstractConnector(383): Stopped ServerConnector@10f9ba48{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-01T20:26:01,752 INFO [M:0;9168bcbe98d9:39089 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:26:01,752 INFO [M:0;9168bcbe98d9:39089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@65507143{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-12-01T20:26:01,752 INFO [M:0;9168bcbe98d9:39089 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7c1597dd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,STOPPED} 2024-12-01T20:26:01,753 INFO [M:0;9168bcbe98d9:39089 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T20:26:01,753 INFO [M:0;9168bcbe98d9:39089 {}] master.HMaster(1770): Stopping master jetty server 2024-12-01T20:26:01,753 INFO [M:0;9168bcbe98d9:39089 {}] server.AbstractConnector(383): Stopped ServerConnector@234e862b{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-01T20:26:01,753 INFO [M:0;9168bcbe98d9:39089 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T20:26:01,754 DEBUG [M:0;9168bcbe98d9:39089 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T20:26:01,754 INFO [M:0;9168bcbe98d9:39089 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-12-01T20:26:01,754 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T20:26:01,754 INFO [M:0;9168bcbe98d9:39089 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;9168bcbe98d9:39089. 2024-12-01T20:26:01,754 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster-HFileCleaner.small.0-1733084659837 {}] cleaner.HFileCleaner(306): Exit Thread[master/9168bcbe98d9:0:becomeActiveMaster-HFileCleaner.small.0-1733084659837,5,FailOnTimeoutGroup] 2024-12-01T20:26:01,754 DEBUG [master/9168bcbe98d9:0:becomeActiveMaster-HFileCleaner.large.0-1733084659837 {}] cleaner.HFileCleaner(306): Exit Thread[master/9168bcbe98d9:0:becomeActiveMaster-HFileCleaner.large.0-1733084659837,5,FailOnTimeoutGroup] 2024-12-01T20:26:01,754 DEBUG [M:0;9168bcbe98d9:39089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:26:01,754 DEBUG [M:0;9168bcbe98d9:39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:26:01,754 DEBUG [M:0;9168bcbe98d9:39089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:26:01,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:26:01,754 DEBUG [M:0;9168bcbe98d9:39089 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T20:26:01,754 INFO [M:0;9168bcbe98d9:39089 {}] hbase.ChoreService(370): Chore service for: master/9168bcbe98d9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T20:26:01,754 INFO [M:0;9168bcbe98d9:39089 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T20:26:01,755 DEBUG [M:0;9168bcbe98d9:39089 {}] master.HMaster(1795): Stopping service threads 2024-12-01T20:26:01,755 INFO [M:0;9168bcbe98d9:39089 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T20:26:01,755 INFO [M:0;9168bcbe98d9:39089 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T20:26:01,755 INFO [M:0;9168bcbe98d9:39089 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T20:26:01,755 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T20:26:01,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-12-01T20:26:01,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-01T20:26:01,756 DEBUG [M:0;9168bcbe98d9:39089 {}] zookeeper.ZKUtil(347): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-12-01T20:26:01,756 WARN [M:0;9168bcbe98d9:39089 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-01T20:26:01,757 INFO [M:0;9168bcbe98d9:39089 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/.lastflushedseqids 2024-12-01T20:26:01,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741981_1157 (size=495) 2024-12-01T20:26:01,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T20:26:01,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101929129790001, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T20:26:01,838 INFO [RS:0;9168bcbe98d9:38993 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T20:26:01,838 INFO [RS:0;9168bcbe98d9:38993 {}] regionserver.HRegionServer(1031): Exiting; stopping=9168bcbe98d9,38993,1733084657496; zookeeper connection closed. 2024-12-01T20:26:01,839 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@22e1f43a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@22e1f43a 2024-12-01T20:26:01,839 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-01T20:26:01,880 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-01T20:26:02,167 INFO [M:0;9168bcbe98d9:39089 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T20:26:02,168 INFO [M:0;9168bcbe98d9:39089 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T20:26:02,169 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T20:26:02,185 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:26:02,185 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:26:02,185 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T20:26:02,185 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:26:02,185 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=318.87 KB heapSize=374.88 KB 2024-12-01T20:26:02,199 DEBUG [M:0;9168bcbe98d9:39089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bb669003502425393d689ca4cd52f8d is 82, key is hbase:meta,,1/info:regioninfo/1733084661153/Put/seqid=0 2024-12-01T20:26:02,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741982_1158 (size=5672) 2024-12-01T20:26:02,603 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bb669003502425393d689ca4cd52f8d 2024-12-01T20:26:02,628 DEBUG [M:0;9168bcbe98d9:39089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ecd8721bec54780bb1158198bf96737 is 2182, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1733084709427/Put/seqid=0 2024-12-01T20:26:02,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741983_1159 (size=26489) 2024-12-01T20:26:03,034 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.31 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ecd8721bec54780bb1158198bf96737 2024-12-01T20:26:03,045 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0ecd8721bec54780bb1158198bf96737 2024-12-01T20:26:03,062 DEBUG [M:0;9168bcbe98d9:39089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f737fa009e24186b6e8fc103a3e6588 is 69, key is 9168bcbe98d9,38993,1733084657496/rs:state/1733084659954/Put/seqid=0 2024-12-01T20:26:03,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741984_1160 (size=5156) 2024-12-01T20:26:03,468 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f737fa009e24186b6e8fc103a3e6588 2024-12-01T20:26:03,478 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1bb669003502425393d689ca4cd52f8d as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1bb669003502425393d689ca4cd52f8d 2024-12-01T20:26:03,484 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1bb669003502425393d689ca4cd52f8d, entries=8, sequenceid=667, filesize=5.5 K 2024-12-01T20:26:03,485 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0ecd8721bec54780bb1158198bf96737 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0ecd8721bec54780bb1158198bf96737 2024-12-01T20:26:03,490 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0ecd8721bec54780bb1158198bf96737 2024-12-01T20:26:03,490 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0ecd8721bec54780bb1158198bf96737, entries=69, sequenceid=667, filesize=25.9 K 2024-12-01T20:26:03,491 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f737fa009e24186b6e8fc103a3e6588 as hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f737fa009e24186b6e8fc103a3e6588 2024-12-01T20:26:03,496 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44065/user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f737fa009e24186b6e8fc103a3e6588, entries=1, sequenceid=667, filesize=5.0 K 2024-12-01T20:26:03,497 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(3140): Finished flush of dataSize ~318.87 KB/326521, heapSize ~374.59 KB/383576, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=667, compaction requested=false 2024-12-01T20:26:03,498 INFO [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:26:03,498 DEBUG [M:0;9168bcbe98d9:39089 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733084762168Disabling compacts and flushes for region at 1733084762168Disabling writes for close at 1733084762185 (+17 ms)Obtaining lock to block concurrent updates at 1733084762185Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733084762185Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326521, getHeapSize=383816, getOffHeapSize=0, getCellsCount=746 at 1733084762185Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733084762186 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733084762186Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733084762198 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733084762198Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733084762609 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733084762628 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733084762628Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733084763046 (+418 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733084763062 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733084763062Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b8b6759: reopening flushed file at 1733084763477 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@732044bd: reopening flushed file at 1733084763485 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@563c23d9: reopening flushed file at 1733084763490 (+5 ms)Finished flush of dataSize ~318.87 KB/326521, heapSize ~374.59 KB/383576, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=667, compaction requested=false at 1733084763497 (+7 ms)Writing region close event to WAL at 1733084763498 (+1 ms)Closed at 1733084763498 2024-12-01T20:26:03,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36971 is added to blk_1073741830_1006 (size=370611) 2024-12-01T20:26:03,501 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/036cc649-b1d5-acdc-cf31-ace1b5c3185e/MasterData/WALs/9168bcbe98d9,39089,1733084655909/9168bcbe98d9%2C39089%2C1733084655909.9168bcbe98d9%2C39089%2C1733084655909.regiongroup-0.1733084658914 not finished, retry = 0 2024-12-01T20:26:03,603 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T20:26:03,603 INFO [M:0;9168bcbe98d9:39089 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T20:26:03,603 INFO [M:0;9168bcbe98d9:39089 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-12-01T20:26:03,604 INFO [M:0;9168bcbe98d9:39089 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39089 2024-12-01T20:26:03,604 INFO [M:0;9168bcbe98d9:39089 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T20:26:03,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T20:26:03,773 INFO [M:0;9168bcbe98d9:39089 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T20:26:03,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39089-0x101929129790000, quorum=127.0.0.1:57617, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T20:26:03,787 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@cb44ad4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T20:26:03,789 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T20:26:03,789 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:26:03,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T20:26:03,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,STOPPED} 2024-12-01T20:26:03,792 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T20:26:03,792 WARN [BP-824697111-172.17.0.3-1733084651933 heartbeating to localhost/127.0.0.1:44065 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T20:26:03,792 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T20:26:03,792 WARN [BP-824697111-172.17.0.3-1733084651933 heartbeating to localhost/127.0.0.1:44065 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-824697111-172.17.0.3-1733084651933 (Datanode Uuid 903de1c7-053c-43f9-b0d3-0c3126b569c3) service to localhost/127.0.0.1:44065 2024-12-01T20:26:03,794 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/data/data1/current/BP-824697111-172.17.0.3-1733084651933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T20:26:03,794 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/cluster_af0e0975-4350-d211-f6c8-fd3352906293/data/data2/current/BP-824697111-172.17.0.3-1733084651933 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T20:26:03,794 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T20:26:03,799 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@38e78c7b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T20:26:03,800 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c336369{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T20:26:03,800 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:26:03,800 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b22dca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T20:26:03,800 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f045f3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,STOPPED} 2024-12-01T20:26:03,808 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T20:26:03,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-01T20:26:03,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-12-01T20:26:03,863 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5bf7740{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-01T20:26:03,864 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f062696{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T20:26:03,864 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:26:03,864 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18552df3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-01T20:26:03,864 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f6ea73d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,STOPPED} 2024-12-01T20:26:03,877 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@482c666e{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-01T20:26:03,878 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e7d9d04{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T20:26:03,878 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:26:03,878 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435b70f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-01T20:26:03,878 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29611ace{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,STOPPED} 2024-12-01T20:26:03,882 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-01T20:26:03,882 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12da9210{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-12-01T20:26:03,882 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@567287a2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T20:26:03,883 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:26:03,883 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31795375{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-01T20:26:03,883 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ccdffea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,STOPPED} 2024-12-01T20:26:03,885 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-12-01T20:26:03,887 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-12-01T20:26:03,887 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-12-01T20:26:03,888 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-01T20:26:03,890 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4aaa7aca{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-12-01T20:26:03,891 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c920e7f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T20:26:03,891 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:26:03,891 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d8c4596{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-01T20:26:03,891 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68f08892{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/80ca12ad-e658-7f1d-648e-0cc1dd6fc1ff/hadoop.log.dir/,STOPPED} 2024-12-01T20:26:03,893 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-01T20:26:03,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped