2024-12-04 15:09:49,685 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-12-04 15:09:49,702 main DEBUG Took 0.014393 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 15:09:49,703 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 15:09:49,703 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 15:09:49,704 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 15:09:49,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,719 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 15:09:49,735 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,738 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,740 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,740 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,741 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,742 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,743 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,744 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,744 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,745 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,746 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,746 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,747 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,747 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,748 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,748 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:09:49,749 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,750 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 15:09:49,752 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:09:49,753 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 15:09:49,755 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 15:09:49,755 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 15:09:49,757 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 15:09:49,757 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 15:09:49,769 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 15:09:49,772 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 15:09:49,773 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 15:09:49,773 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 15:09:49,774 main DEBUG createAppenders(={Console}) 2024-12-04 15:09:49,775 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-12-04 15:09:49,775 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-12-04 15:09:49,775 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-12-04 15:09:49,776 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 15:09:49,776 main DEBUG OutputStream closed 2024-12-04 15:09:49,776 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 15:09:49,776 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 15:09:49,777 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-12-04 15:09:49,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 15:09:49,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 15:09:49,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 15:09:49,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 15:09:49,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 15:09:49,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 15:09:49,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 15:09:49,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 15:09:49,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 15:09:49,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 15:09:49,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 15:09:49,850 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 15:09:49,850 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 15:09:49,850 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 15:09:49,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 15:09:49,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 15:09:49,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 15:09:49,852 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 15:09:49,854 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 15:09:49,855 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-12-04 15:09:49,855 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 15:09:49,856 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-12-04T15:09:49,892 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-12-04 15:09:49,894 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 15:09:49,895 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T15:09:50,207 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a 2024-12-04T15:09:50,211 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-12-04T15:09:50,213 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:09:50,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T15:09:50,228 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322, deleteOnExit=true 2024-12-04T15:09:50,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T15:09:50,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/test.cache.data in system properties and HBase conf 2024-12-04T15:09:50,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T15:09:50,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir in system properties and HBase conf 2024-12-04T15:09:50,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T15:09:50,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T15:09:50,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T15:09:50,320 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T15:09:50,417 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T15:09:50,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T15:09:50,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T15:09:50,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T15:09:50,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T15:09:50,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T15:09:50,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T15:09:50,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T15:09:50,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T15:09:50,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T15:09:50,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/nfs.dump.dir in system properties and HBase conf 2024-12-04T15:09:50,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/java.io.tmpdir in system properties and HBase conf 2024-12-04T15:09:50,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T15:09:50,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T15:09:50,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T15:09:51,331 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T15:09:51,417 INFO [Time-limited test {}] log.Log(170): Logging initialized @2590ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T15:09:51,535 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:09:51,641 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T15:09:51,672 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:09:51,672 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:09:51,674 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T15:09:51,693 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:09:51,695 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57594f23{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:09:51,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13488ad7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T15:09:51,942 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@41b74231{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/java.io.tmpdir/jetty-localhost-43229-hadoop-hdfs-3_4_1-tests_jar-_-any-1220528721784121404/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T15:09:51,956 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1cfa68d8{HTTP/1.1, (http/1.1)}{localhost:43229} 2024-12-04T15:09:51,956 INFO [Time-limited test {}] server.Server(415): Started @3130ms 2024-12-04T15:09:52,400 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:09:52,407 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T15:09:52,408 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:09:52,408 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:09:52,408 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T15:09:52,409 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b27bdff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:09:52,410 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a4f7ff0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T15:09:52,531 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79fa0fb2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/java.io.tmpdir/jetty-localhost-36705-hadoop-hdfs-3_4_1-tests_jar-_-any-9695742941094024584/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T15:09:52,532 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@482cd6b5{HTTP/1.1, (http/1.1)}{localhost:36705} 2024-12-04T15:09:52,532 INFO [Time-limited test {}] server.Server(415): Started @3706ms 2024-12-04T15:09:52,595 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T15:09:53,146 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/data/data2/current/BP-2007428057-172.17.0.2-1733324991063/current, will proceed with Du for space computation calculation, 2024-12-04T15:09:53,146 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/data/data1/current/BP-2007428057-172.17.0.2-1733324991063/current, will proceed with Du for space computation calculation, 2024-12-04T15:09:53,191 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T15:09:53,250 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8cc91132263b7019 with lease ID 0xc0ca84a7a13bd058: Processing first storage report for DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037 from datanode DatanodeRegistration(127.0.0.1:38861, datanodeUuid=5aede1a2-d9bc-4774-9318-20053b7244bc, infoPort=38655, infoSecurePort=0, ipcPort=38523, storageInfo=lv=-57;cid=testClusterID;nsid=1698579251;c=1733324991063) 2024-12-04T15:09:53,251 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8cc91132263b7019 with lease ID 0xc0ca84a7a13bd058: from storage DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037 node DatanodeRegistration(127.0.0.1:38861, datanodeUuid=5aede1a2-d9bc-4774-9318-20053b7244bc, infoPort=38655, infoSecurePort=0, ipcPort=38523, storageInfo=lv=-57;cid=testClusterID;nsid=1698579251;c=1733324991063), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T15:09:53,252 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8cc91132263b7019 with lease ID 0xc0ca84a7a13bd058: Processing first storage report for DS-84b353d3-9f68-4016-9d42-9147dc27be0c from datanode DatanodeRegistration(127.0.0.1:38861, datanodeUuid=5aede1a2-d9bc-4774-9318-20053b7244bc, infoPort=38655, infoSecurePort=0, ipcPort=38523, storageInfo=lv=-57;cid=testClusterID;nsid=1698579251;c=1733324991063) 2024-12-04T15:09:53,252 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8cc91132263b7019 with lease ID 0xc0ca84a7a13bd058: from storage DS-84b353d3-9f68-4016-9d42-9147dc27be0c node DatanodeRegistration(127.0.0.1:38861, datanodeUuid=5aede1a2-d9bc-4774-9318-20053b7244bc, infoPort=38655, infoSecurePort=0, ipcPort=38523, storageInfo=lv=-57;cid=testClusterID;nsid=1698579251;c=1733324991063), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T15:09:53,323 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a 2024-12-04T15:09:53,407 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/zookeeper_0, clientPort=50985, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T15:09:53,419 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50985 2024-12-04T15:09:53,434 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:09:53,438 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:09:53,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741825_1001 (size=7) 2024-12-04T15:09:54,080 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 with version=8 2024-12-04T15:09:54,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/hbase-staging 2024-12-04T15:09:54,093 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-12-04T15:09:54,093 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-12-04T15:09:54,093 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-12-04T15:09:54,093 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-12-04T15:09:54,175 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T15:09:54,441 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ff648bb9ef17:0 server-side Connection retries=45 2024-12-04T15:09:54,453 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T15:09:54,453 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T15:09:54,458 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T15:09:54,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T15:09:54,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T15:09:54,622 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T15:09:54,690 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T15:09:54,699 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T15:09:54,704 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T15:09:54,736 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 22877 (auto-detected) 2024-12-04T15:09:54,737 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T15:09:54,756 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42357 2024-12-04T15:09:54,779 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42357 connecting to ZooKeeper ensemble=127.0.0.1:50985 2024-12-04T15:09:54,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423570x0, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T15:09:54,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42357-0x1017e065dfa0000 connected 2024-12-04T15:09:54,852 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:09:54,854 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:09:54,864 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-04T15:09:54,886 INFO [Time-limited test {}] log.Log(170): Logging initialized @6060ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T15:09:54,934 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-12-04T15:09:54,934 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-12-04T15:09:54,935 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-12-04T15:09:54,935 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-12-04T15:09:54,936 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-12-04T15:09:54,938 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-12-04T15:09:54,938 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-12-04T15:09:54,938 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-12-04T15:09:54,940 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-12-04T15:09:55,002 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 36847 2024-12-04T15:09:55,003 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-04T15:09:55,017 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T15:09:55,018 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@16164b0d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:09:55,019 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T15:09:55,019 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4f87c64{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-12-04T15:09:55,058 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-12-04T15:09:55,066 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:09:55,066 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:09:55,067 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T15:09:55,072 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T15:09:55,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@e084f9e{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-12-04T15:09:55,762 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@fd624d{HTTP/1.1, (http/1.1)}{0.0.0.0:36847} 2024-12-04T15:09:55,762 INFO [Time-limited test {}] server.Server(415): Started @6936ms 2024-12-04T15:09:55,766 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48, hbase.cluster.distributed=false 2024-12-04T15:09:55,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-12-04T15:09:55,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42357 2024-12-04T15:09:55,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42357 2024-12-04T15:09:55,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42357 2024-12-04T15:09:55,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42357 2024-12-04T15:09:55,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42357 2024-12-04T15:09:55,919 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ff648bb9ef17:0 server-side Connection retries=45 2024-12-04T15:09:55,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T15:09:55,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T15:09:55,922 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T15:09:55,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T15:09:55,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T15:09:55,926 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T15:09:55,929 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T15:09:55,930 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41657 2024-12-04T15:09:55,932 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41657 connecting to ZooKeeper ensemble=127.0.0.1:50985 2024-12-04T15:09:55,933 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:09:55,937 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:09:55,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416570x0, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T15:09:55,944 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41657-0x1017e065dfa0001 connected 2024-12-04T15:09:55,944 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-04T15:09:55,946 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-12-04T15:09:55,946 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-12-04T15:09:55,947 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-12-04T15:09:55,947 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-12-04T15:09:55,947 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-12-04T15:09:55,949 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-12-04T15:09:55,949 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-12-04T15:09:55,950 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-12-04T15:09:55,950 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-12-04T15:09:55,952 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 35639 2024-12-04T15:09:55,952 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-04T15:09:55,954 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T15:09:55,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3f822913{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:09:55,955 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T15:09:55,955 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6e82f00d{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-12-04T15:09:55,968 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-12-04T15:09:55,969 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:09:55,969 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:09:55,969 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T15:09:55,970 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-12-04T15:09:55,974 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@530459fd{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-12-04T15:09:55,974 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21b68432{HTTP/1.1, (http/1.1)}{0.0.0.0:35639} 2024-12-04T15:09:55,975 INFO [Time-limited test {}] server.Server(415): Started @7149ms 2024-12-04T15:09:55,979 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T15:09:55,988 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T15:09:55,991 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-12-04T15:09:55,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-12-04T15:09:55,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41657 2024-12-04T15:09:55,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41657 2024-12-04T15:09:55,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41657 2024-12-04T15:09:56,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41657 2024-12-04T15:09:56,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41657 2024-12-04T15:09:56,018 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ff648bb9ef17:42357 2024-12-04T15:09:56,021 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-12-04T15:09:56,023 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@701715b7{HTTP/1.1, (http/1.1)}{0.0.0.0:45939} 2024-12-04T15:09:56,023 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] server.Server(415): Started @7197ms 2024-12-04T15:09:56,024 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:56,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-04T15:09:56,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-04T15:09:56,033 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:56,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-12-04T15:09:56,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:56,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:56,057 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/master 2024-12-04T15:09:56,059 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/ff648bb9ef17,42357,1733324994229 from backup master directory 2024-12-04T15:09:56,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:56,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-04T15:09:56,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-12-04T15:09:56,064 WARN [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T15:09:56,064 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:56,066 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T15:09:56,068 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T15:09:56,128 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/hbase.id] with ID: ce837f02-95e7-4231-a6b7-9a4367546947 2024-12-04T15:09:56,128 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.tmp/hbase.id 2024-12-04T15:09:56,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741826_1002 (size=42) 2024-12-04T15:09:56,542 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.tmp/hbase.id]:[hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/hbase.id] 2024-12-04T15:09:56,587 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:09:56,591 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T15:09:56,612 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-04T15:09:56,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:56,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:56,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741827_1003 (size=196) 2024-12-04T15:09:56,646 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:09:56,648 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T15:09:56,654 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-04T15:09:56,656 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-04T15:09:56,667 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:09:56,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741828_1004 (size=1189) 2024-12-04T15:09:57,121 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store 2024-12-04T15:09:57,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741829_1005 (size=34) 2024-12-04T15:09:57,546 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T15:09:57,549 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:09:57,550 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T15:09:57,550 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:09:57,550 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:09:57,552 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T15:09:57,552 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:09:57,552 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:09:57,553 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733324997550Disabling compacts and flushes for region at 1733324997550Disabling writes for close at 1733324997552 (+2 ms)Writing region close event to WAL at 1733324997552Closed at 1733324997552 2024-12-04T15:09:57,555 WARN [master/ff648bb9ef17:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/.initializing 2024-12-04T15:09:57,556 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/WALs/ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:57,560 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T15:09:57,565 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor ff648bb9ef17,42357,1733324994229.regiongroup-0 2024-12-04T15:09:57,580 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ff648bb9ef17%2C42357%2C1733324994229.ff648bb9ef17%2C42357%2C1733324994229.regiongroup-0, suffix=, logDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/WALs/ff648bb9ef17,42357,1733324994229, archiveDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/oldWALs, maxLogs=10 2024-12-04T15:09:57,606 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/WALs/ff648bb9ef17,42357,1733324994229/ff648bb9ef17%2C42357%2C1733324994229.ff648bb9ef17%2C42357%2C1733324994229.regiongroup-0.1733324997584, exclude list is [], retry=0 2024-12-04T15:09:57,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:09:57,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T15:09:57,663 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/WALs/ff648bb9ef17,42357,1733324994229/ff648bb9ef17%2C42357%2C1733324994229.ff648bb9ef17%2C42357%2C1733324994229.regiongroup-0.1733324997584 2024-12-04T15:09:57,664 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:09:57,665 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:09:57,665 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:09:57,669 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,669 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T15:09:57,743 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:57,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:57,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T15:09:57,750 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:57,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:09:57,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T15:09:57,754 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:57,754 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:09:57,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T15:09:57,757 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:57,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:09:57,758 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,761 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,762 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,768 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,768 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,772 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T15:09:57,775 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:09:57,779 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:09:57,780 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9821902080, jitterRate=-0.08526408672332764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T15:09:57,786 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733324997686Initializing all the Stores at 1733324997689 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324997689Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324997691 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324997691Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324997691Cleaning up temporary data from old regions at 1733324997768 (+77 ms)Region opened successfully at 1733324997786 (+18 ms) 2024-12-04T15:09:57,787 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T15:09:57,817 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ae020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=ff648bb9ef17/172.17.0.2:0 2024-12-04T15:09:57,851 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T15:09:57,863 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T15:09:57,863 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T15:09:57,866 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T15:09:57,868 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T15:09:57,873 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T15:09:57,873 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T15:09:57,899 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T15:09:57,916 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-12-04T15:09:57,918 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-12-04T15:09:57,921 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T15:09:57,923 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-12-04T15:09:57,924 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-12-04T15:09:57,926 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T15:09:57,930 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-12-04T15:09:57,933 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-12-04T15:09:57,934 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-12-04T15:09:57,936 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-12-04T15:09:57,961 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T15:09:57,962 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-12-04T15:09:57,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-12-04T15:09:57,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-12-04T15:09:57,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:57,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:57,970 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ff648bb9ef17,42357,1733324994229, sessionid=0x1017e065dfa0000, setting cluster-up flag (Was=false) 2024-12-04T15:09:58,003 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-12-04T15:09:58,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:58,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:58,024 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-12-04T15:09:58,026 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:58,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:58,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:58,041 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-12-04T15:09:58,043 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:58,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:58,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:58,056 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-12-04T15:09:58,058 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:58,071 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T15:09:58,112 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(746): ClusterId : ce837f02-95e7-4231-a6b7-9a4367546947 2024-12-04T15:09:58,114 INFO [RS:0;ff648bb9ef17:41657 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-12-04T15:09:58,116 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T15:09:58,123 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T15:09:58,123 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-12-04T15:09:58,127 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-12-04T15:09:58,127 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T15:09:58,130 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T15:09:58,131 DEBUG [RS:0;ff648bb9ef17:41657 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d855b8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=ff648bb9ef17/172.17.0.2:0 2024-12-04T15:09:58,156 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ff648bb9ef17:41657 2024-12-04T15:09:58,160 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T15:09:58,160 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T15:09:58,160 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T15:09:58,163 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(2659): reportForDuty to master=ff648bb9ef17,42357,1733324994229 with port=41657, startcode=1733324995877 2024-12-04T15:09:58,174 DEBUG [RS:0;ff648bb9ef17:41657 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T15:09:58,176 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T15:09:58,188 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T15:09:58,196 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T15:09:58,203 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ff648bb9ef17,42357,1733324994229 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T15:09:58,214 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ff648bb9ef17:0, corePoolSize=5, maxPoolSize=5 2024-12-04T15:09:58,214 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ff648bb9ef17:0, corePoolSize=5, maxPoolSize=5 2024-12-04T15:09:58,214 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38741, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T15:09:58,214 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ff648bb9ef17:0, corePoolSize=5, maxPoolSize=5 2024-12-04T15:09:58,214 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ff648bb9ef17:0, corePoolSize=5, maxPoolSize=5 2024-12-04T15:09:58,215 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ff648bb9ef17:0, corePoolSize=10, maxPoolSize=10 2024-12-04T15:09:58,215 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,215 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ff648bb9ef17:0, corePoolSize=2, maxPoolSize=2 2024-12-04T15:09:58,215 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,219 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733325028219 2024-12-04T15:09:58,221 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T15:09:58,221 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T15:09:58,221 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42357 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:09:58,221 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T15:09:58,222 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T15:09:58,226 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T15:09:58,226 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-12-04T15:09:58,226 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T15:09:58,227 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T15:09:58,227 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T15:09:58,229 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:58,229 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T15:09:58,229 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,234 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T15:09:58,235 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T15:09:58,235 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T15:09:58,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741831_1007 (size=1321) 2024-12-04T15:09:58,242 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T15:09:58,243 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:09:58,249 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465c71d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:09:58,249 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:09:58,253 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:09:58,253 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T15:09:58,253 WARN [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T15:09:58,259 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:09:58,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741832_1008 (size=32) 2024-12-04T15:09:58,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:09:58,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:09:58,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b54638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:09:58,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:09:58,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:09:58,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:09:58,274 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:09:58,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c90f8c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:09:58,277 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-12-04T15:09:58,279 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T15:09:58,279 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T15:09:58,281 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ff648bb9ef17:0:becomeActiveMaster-HFileCleaner.large.0-1733324998281,5,FailOnTimeoutGroup] 2024-12-04T15:09:58,282 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ff648bb9ef17:0:becomeActiveMaster-HFileCleaner.small.0-1733324998281,5,FailOnTimeoutGroup] 2024-12-04T15:09:58,282 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,282 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T15:09:58,283 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,283 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,355 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(2659): reportForDuty to master=ff648bb9ef17,42357,1733324994229 with port=41657, startcode=1733324995877 2024-12-04T15:09:58,357 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:58,359 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42357 {}] master.ServerManager(517): Registering regionserver=ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:58,367 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:09:58,367 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36003 2024-12-04T15:09:58,368 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=36847 2024-12-04T15:09:58,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-12-04T15:09:58,376 DEBUG [RS:0;ff648bb9ef17:41657 {}] zookeeper.ZKUtil(111): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rs/ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:58,377 WARN [RS:0;ff648bb9ef17:41657 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T15:09:58,377 INFO [RS:0;ff648bb9ef17:41657 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-04T15:09:58,377 INFO [RS:0;ff648bb9ef17:41657 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-04T15:09:58,377 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:58,379 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ff648bb9ef17,41657,1733324995877] 2024-12-04T15:09:58,403 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T15:09:58,417 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T15:09:58,423 INFO [RS:0;ff648bb9ef17:41657 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T15:09:58,423 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,424 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T15:09:58,430 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T15:09:58,432 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,432 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,432 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,432 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,432 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,432 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,433 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ff648bb9ef17:0, corePoolSize=2, maxPoolSize=2 2024-12-04T15:09:58,433 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,433 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,433 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,433 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,433 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,433 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ff648bb9ef17:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:09:58,433 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0, corePoolSize=3, maxPoolSize=3 2024-12-04T15:09:58,434 DEBUG [RS:0;ff648bb9ef17:41657 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ff648bb9ef17:0, corePoolSize=3, maxPoolSize=3 2024-12-04T15:09:58,434 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,435 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,435 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,435 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,435 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,435 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ff648bb9ef17,41657,1733324995877-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T15:09:58,459 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T15:09:58,461 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ff648bb9ef17,41657,1733324995877-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,461 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,461 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.Replication(171): ff648bb9ef17,41657,1733324995877 started 2024-12-04T15:09:58,479 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:58,479 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1482): Serving as ff648bb9ef17,41657,1733324995877, RpcServer on ff648bb9ef17/172.17.0.2:41657, sessionid=0x1017e065dfa0001 2024-12-04T15:09:58,480 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T15:09:58,480 DEBUG [RS:0;ff648bb9ef17:41657 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:58,480 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ff648bb9ef17,41657,1733324995877' 2024-12-04T15:09:58,481 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-12-04T15:09:58,481 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-12-04T15:09:58,482 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T15:09:58,482 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-12-04T15:09:58,482 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ff648bb9ef17,41657,1733324995877' 2024-12-04T15:09:58,482 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:09:58,483 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:09:58,483 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-12-04T15:09:58,484 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-12-04T15:09:58,484 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T15:09:58,484 DEBUG [RS:0;ff648bb9ef17:41657 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:58,484 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ff648bb9ef17,41657,1733324995877' 2024-12-04T15:09:58,484 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-12-04T15:09:58,484 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-12-04T15:09:58,485 DEBUG [RS:0;ff648bb9ef17:41657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T15:09:58,485 INFO [RS:0;ff648bb9ef17:41657 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T15:09:58,485 INFO [RS:0;ff648bb9ef17:41657 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T15:09:58,590 INFO [RS:0;ff648bb9ef17:41657 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T15:09:58,591 INFO [RS:0;ff648bb9ef17:41657 {}] monitor.StreamSlowMonitor(122): New stream slow monitor ff648bb9ef17,41657,1733324995877.regiongroup-0 2024-12-04T15:09:58,594 INFO [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0, suffix=, logDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877, archiveDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs, maxLogs=32 2024-12-04T15:09:58,610 DEBUG [RS:0;ff648bb9ef17:41657 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733324998596, exclude list is [], retry=0 2024-12-04T15:09:58,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:09:58,618 INFO [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733324998596 2024-12-04T15:09:58,618 DEBUG [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:09:58,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:09:58,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T15:09:58,672 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T15:09:58,672 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:58,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:58,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T15:09:58,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T15:09:58,676 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:58,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:58,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T15:09:58,679 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T15:09:58,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:58,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:58,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T15:09:58,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T15:09:58,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:58,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:58,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T15:09:58,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740 2024-12-04T15:09:58,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740 2024-12-04T15:09:58,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T15:09:58,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T15:09:58,690 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T15:09:58,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T15:09:58,697 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:09:58,698 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11090235680, jitterRate=0.03285868465900421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T15:09:58,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733324998666Initializing all the Stores at 1733324998668 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324998669 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324998669Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324998669Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324998669Cleaning up temporary data from old regions at 1733324998689 (+20 ms)Region opened successfully at 1733324998701 (+12 ms) 2024-12-04T15:09:58,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T15:09:58,702 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T15:09:58,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T15:09:58,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T15:09:58,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T15:09:58,703 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T15:09:58,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733324998702Disabling compacts and flushes for region at 1733324998702Disabling writes for close at 1733324998702Writing region close event to WAL at 1733324998703 (+1 ms)Closed at 1733324998703 2024-12-04T15:09:58,707 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T15:09:58,707 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T15:09:58,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T15:09:58,722 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T15:09:58,725 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T15:09:58,877 DEBUG [ff648bb9ef17:42357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T15:09:58,893 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:58,900 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ff648bb9ef17,41657,1733324995877, state=OPENING 2024-12-04T15:09:58,905 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T15:09:58,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:58,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:09:58,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-04T15:09:58,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-04T15:09:58,909 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T15:09:58,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:09:59,089 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T15:09:59,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50705, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T15:09:59,104 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T15:09:59,105 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-12-04T15:09:59,105 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-12-04T15:09:59,105 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T15:09:59,105 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T15:09:59,108 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ff648bb9ef17%2C41657%2C1733324995877.meta, suffix=.meta, logDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877, archiveDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs, maxLogs=32 2024-12-04T15:09:59,123 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733324999110.meta, exclude list is [], retry=0 2024-12-04T15:09:59,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:09:59,130 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733324999110.meta 2024-12-04T15:09:59,130 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:09:59,131 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:09:59,135 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:09:59,136 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T15:09:59,139 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T15:09:59,139 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T15:09:59,142 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T15:09:59,143 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:09:59,143 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T15:09:59,143 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T15:09:59,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T15:09:59,148 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T15:09:59,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:59,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:59,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T15:09:59,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T15:09:59,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:59,151 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:59,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T15:09:59,153 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T15:09:59,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:59,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:59,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T15:09:59,155 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T15:09:59,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:09:59,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:09:59,156 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T15:09:59,157 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740 2024-12-04T15:09:59,159 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740 2024-12-04T15:09:59,162 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T15:09:59,162 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T15:09:59,163 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T15:09:59,165 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T15:09:59,166 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9534979840, jitterRate=-0.11198580265045166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T15:09:59,167 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T15:09:59,168 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733324999144Writing region info on filesystem at 1733324999144Initializing all the Stores at 1733324999146 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324999146Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324999146Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733324999147 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733324999147Cleaning up temporary data from old regions at 1733324999162 (+15 ms)Running coprocessor post-open hooks at 1733324999167 (+5 ms)Region opened successfully at 1733324999168 (+1 ms) 2024-12-04T15:09:59,175 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733324999077 2024-12-04T15:09:59,187 DEBUG [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T15:09:59,187 INFO [RS_OPEN_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T15:09:59,189 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:59,191 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ff648bb9ef17,41657,1733324995877, state=OPEN 2024-12-04T15:09:59,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-12-04T15:09:59,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-12-04T15:09:59,193 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-04T15:09:59,193 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-12-04T15:09:59,194 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ff648bb9ef17,41657,1733324995877 2024-12-04T15:09:59,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T15:09:59,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ff648bb9ef17,41657,1733324995877 in 284 msec 2024-12-04T15:09:59,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T15:09:59,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 487 msec 2024-12-04T15:09:59,206 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T15:09:59,206 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T15:09:59,225 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:09:59,226 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:09:59,247 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:09:59,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56009, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:09:59,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1610 sec 2024-12-04T15:09:59,276 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733324999275, completionTime=-1 2024-12-04T15:09:59,279 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T15:09:59,279 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T15:09:59,313 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T15:09:59,313 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733325059313 2024-12-04T15:09:59,313 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733325119313 2024-12-04T15:09:59,313 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 33 msec 2024-12-04T15:09:59,315 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ff648bb9ef17,42357,1733324994229-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:59,316 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ff648bb9ef17,42357,1733324994229-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:59,316 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ff648bb9ef17,42357,1733324994229-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:59,318 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ff648bb9ef17:42357, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:59,318 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:59,319 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:59,325 DEBUG [master/ff648bb9ef17:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T15:09:59,354 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.290sec 2024-12-04T15:09:59,355 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T15:09:59,357 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T15:09:59,358 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T15:09:59,359 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T15:09:59,359 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T15:09:59,360 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ff648bb9ef17,42357,1733324994229-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T15:09:59,361 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ff648bb9ef17,42357,1733324994229-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T15:09:59,370 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T15:09:59,371 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T15:09:59,372 INFO [master/ff648bb9ef17:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ff648bb9ef17,42357,1733324994229-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:09:59,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30388da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:09:59,418 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T15:09:59,418 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T15:09:59,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:09:59,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:09:59,431 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:09:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:09:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:09:59,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@312033b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:09:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:09:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:09:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:09:59,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:09:59,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@707a8337, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:09:59,438 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:09:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:09:59,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:09:59,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:09:59,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ff648bb9ef17,42357,1733324994229 2024-12-04T15:09:59,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-12-04T15:09:59,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/test.cache.data in system properties and HBase conf 2024-12-04T15:09:59,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T15:09:59,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir in system properties and HBase conf 2024-12-04T15:09:59,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T15:09:59,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T15:09:59,476 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T15:09:59,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T15:09:59,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T15:09:59,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T15:09:59,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T15:09:59,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T15:09:59,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T15:09:59,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T15:09:59,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T15:09:59,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T15:09:59,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/nfs.dump.dir in system properties and HBase conf 2024-12-04T15:09:59,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/java.io.tmpdir in system properties and HBase conf 2024-12-04T15:09:59,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T15:09:59,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T15:09:59,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T15:09:59,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741835_1011 (size=592039) 2024-12-04T15:09:59,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741836_1012 (size=1663647) 2024-12-04T15:10:01,088 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:10:01,192 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:10:01,388 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T15:10:01,394 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:10:01,394 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:10:01,394 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T15:10:01,398 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-04T15:10:01,399 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T15:10:01,400 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e8f084e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:10:01,400 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5fbf2bd8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-04T15:10:01,402 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:10:01,402 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:10:01,402 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T15:10:01,405 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:10:01,417 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@382efc4a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:10:01,418 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48ed5d50{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-04T15:10:01,533 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2024-12-04T15:10:01,533 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2024-12-04T15:10:01,534 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-04T15:10:01,535 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-04T15:10:01,576 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:01,687 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:01,902 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:01,921 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@39087899{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/java.io.tmpdir/jetty-localhost-35421-hadoop-yarn-common-3_4_1_jar-_-any-18335203949736513237/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-12-04T15:10:01,921 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@db534bf{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/java.io.tmpdir/jetty-localhost-33111-hadoop-yarn-common-3_4_1_jar-_-any-8126983647384728746/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-12-04T15:10:01,921 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@99937f3{HTTP/1.1, (http/1.1)}{localhost:35421} 2024-12-04T15:10:01,922 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4aa9facb{HTTP/1.1, (http/1.1)}{localhost:33111} 2024-12-04T15:10:01,922 INFO [Thread-251 {}] server.Server(415): Started @13096ms 2024-12-04T15:10:01,922 INFO [Time-limited test {}] server.Server(415): Started @13096ms 2024-12-04T15:10:02,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741837_1013 (size=5) 2024-12-04T15:10:03,125 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-12-04T15:10:03,130 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:10:03,160 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-04T15:10:03,161 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T15:10:03,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:10:03,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:10:03,165 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T15:10:03,168 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:10:03,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@404f25b1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:10:03,170 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69ddd664{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-04T15:10:03,229 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-12-04T15:10:03,229 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-04T15:10:03,229 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-12-04T15:10:03,229 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-04T15:10:03,238 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:03,245 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:03,343 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:03,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a9ede50{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/java.io.tmpdir/jetty-localhost-35929-hadoop-yarn-common-3_4_1_jar-_-any-15237993086306533837/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-04T15:10:03,353 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@27800845{HTTP/1.1, (http/1.1)}{localhost:35929} 2024-12-04T15:10:03,354 INFO [Time-limited test {}] server.Server(415): Started @14528ms 2024-12-04T15:10:03,484 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-12-04T15:10:03,487 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:10:03,499 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-12-04T15:10:03,499 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T15:10:03,503 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:10:03,503 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:10:03,503 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T15:10:03,504 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:10:03,506 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59559291{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:10:03,507 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e8f6b9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-12-04T15:10:03,561 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-12-04T15:10:03,561 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-12-04T15:10:03,561 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-12-04T15:10:03,561 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-12-04T15:10:03,568 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:03,572 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:03,646 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-12-04T15:10:03,651 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b3d815c{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/java.io.tmpdir/jetty-localhost-36311-hadoop-yarn-common-3_4_1_jar-_-any-5380944834192950776/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-04T15:10:03,651 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@33598bd6{HTTP/1.1, (http/1.1)}{localhost:36311} 2024-12-04T15:10:03,651 INFO [Time-limited test {}] server.Server(415): Started @14826ms 2024-12-04T15:10:03,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-12-04T15:10:03,680 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:36003/backupUT 2024-12-04T15:10:03,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:10:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ff648bb9ef17,42357,1733324994229 2024-12-04T15:10:03,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a1c5991 2024-12-04T15:10:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:10:03,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T15:10:03,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-12-04T15:10:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-12-04T15:10:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T15:10:03,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 20 msec 2024-12-04T15:10:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T15:10:03,823 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-12-04T15:10:03,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-12-04T15:10:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-12-04T15:10:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-12-04T15:10:03,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 13 msec 2024-12-04T15:10:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-12-04T15:10:03,943 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-12-04T15:10:03,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-12-04T15:10:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-12-04T15:10:03,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-12-04T15:10:03,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 12 msec 2024-12-04T15:10:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-12-04T15:10:04,064 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-12-04T15:10:04,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-12-04T15:10:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-12-04T15:10:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T15:10:04,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 11 msec 2024-12-04T15:10:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T15:10:04,174 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-12-04T15:10:04,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1733325003680', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1733325003680 2024-12-04T15:10:04,194 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1733325003680 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:10:04,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1733325003680" procId is: 8 2024-12-04T15:10:04,197 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T15:10:04,199 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1733325003680 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:10:04,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741838_1014 (size=367) 2024-12-04T15:10:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T15:10:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T15:10:04,614 INFO [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 44b146fc21bd97567dc283c76bd9e484, NAME => 'test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1733325003680', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:10:04,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741839_1015 (size=53) 2024-12-04T15:10:04,625 DEBUG [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(898): Instantiated test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:04,625 DEBUG [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(1722): Closing 44b146fc21bd97567dc283c76bd9e484, disabling compactions & flushes 2024-12-04T15:10:04,625 INFO [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(1755): Closing region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:04,625 DEBUG [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:04,625 DEBUG [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. after waiting 0 ms 2024-12-04T15:10:04,625 DEBUG [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:04,625 INFO [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(1973): Closed test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:04,625 DEBUG [RegionOpenAndInit-test-1733325003680-pool-0 {}] regionserver.HRegion(1676): Region close journal for 44b146fc21bd97567dc283c76bd9e484: Waiting for close lock at 1733325004625Disabling compacts and flushes for region at 1733325004625Disabling writes for close at 1733325004625Writing region close event to WAL at 1733325004625Closed at 1733325004625 2024-12-04T15:10:04,627 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1733325003680 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:10:04,633 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733325004628"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733325004628"}]},"ts":"1733325004628"} 2024-12-04T15:10:04,644 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:10:04,647 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1733325003680 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:10:04,651 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1733325003680","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325004647"}]},"ts":"1733325004647"} 2024-12-04T15:10:04,652 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-04T15:10:04,657 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1733325003680, state=ENABLING in hbase:meta 2024-12-04T15:10:04,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, ASSIGN}] 2024-12-04T15:10:04,663 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, ASSIGN 2024-12-04T15:10:04,665 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, ASSIGN; state=OFFLINE, location=ff648bb9ef17,41657,1733324995877; forceNewPlan=false, retain=false 2024-12-04T15:10:04,712 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T15:10:04,715 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T15:10:04,715 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T15:10:04,718 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:10:04,718 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-12-04T15:10:04,718 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T15:10:04,718 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T15:10:04,719 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T15:10:04,719 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T15:10:04,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:04,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, ASSIGN because future has completed 2024-12-04T15:10:04,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T15:10:04,998 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:04,999 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T15:10:04,999 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor ff648bb9ef17,41657,1733324995877.regiongroup-1 2024-12-04T15:10:05,002 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1, suffix=, logDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877, archiveDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs, maxLogs=32 2024-12-04T15:10:05,019 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325005004, exclude list is [], retry=0 2024-12-04T15:10:05,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:05,025 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325005004 2024-12-04T15:10:05,026 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:05,026 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 44b146fc21bd97567dc283c76bd9e484, NAME => 'test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:10:05,027 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:10:05,027 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733325003680 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,027 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:05,027 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,027 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,031 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,033 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44b146fc21bd97567dc283c76bd9e484 columnFamilyName f 2024-12-04T15:10:05,033 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:05,034 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(327): Store=44b146fc21bd97567dc283c76bd9e484/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:05,034 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,036 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,036 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,037 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,037 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,040 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,045 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:10:05,046 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 44b146fc21bd97567dc283c76bd9e484; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9925788640, jitterRate=-0.0755888968706131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T15:10:05,046 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:05,047 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 44b146fc21bd97567dc283c76bd9e484: Running coprocessor pre-open hook at 1733325005027Writing region info on filesystem at 1733325005027Initializing all the Stores at 1733325005029 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325005029Cleaning up temporary data from old regions at 1733325005037 (+8 ms)Running coprocessor post-open hooks at 1733325005046 (+9 ms)Region opened successfully at 1733325005047 (+1 ms) 2024-12-04T15:10:05,049 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., pid=10, masterSystemTime=1733325004975 2024-12-04T15:10:05,054 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:05,054 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:05,055 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=OPEN, openSeqNum=2, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:05,056 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42357 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ff648bb9ef17,41657,1733324995877, table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T15:10:05,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:05,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T15:10:05,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 in 239 msec 2024-12-04T15:10:05,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-04T15:10:05,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, ASSIGN in 406 msec 2024-12-04T15:10:05,071 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1733325003680 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:10:05,071 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1733325003680","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325005071"}]},"ts":"1733325005071"} 2024-12-04T15:10:05,074 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1733325003680, state=ENABLED in hbase:meta 2024-12-04T15:10:05,075 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1733325003680 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:10:05,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1733325003680 in 889 msec 2024-12-04T15:10:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-12-04T15:10:05,333 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1733325003680 completed 2024-12-04T15:10:05,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ddd074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:05,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:05,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:05,336 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3802ce18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:05,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:05,339 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526d938f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:05,343 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:05,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:05,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733325003680', row='row0', locateType=CURRENT is [region=test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., hostname=ff648bb9ef17,41657,1733324995877, seqNum=2] 2024-12-04T15:10:05,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41657 {}] regionserver.HRegion(8528): writing data to region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. with WAL disabled. Data may be lost in the event of a crash. 2024-12-04T15:10:05,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17333250036801', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17333250036801 2024-12-04T15:10:05,505 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17333250036801 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:10:05,506 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:05,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17333250036801" procId is: 11 2024-12-04T15:10:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:10:05,507 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17333250036801 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:10:05,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741841_1017 (size=364) 2024-12-04T15:10:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:10:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:10:05,918 INFO [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0896086d7d737ceb1077211e1712c9da, NAME => 'ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17333250036801', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:10:05,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741842_1018 (size=50) 2024-12-04T15:10:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:10:06,328 DEBUG [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:06,328 DEBUG [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(1722): Closing 0896086d7d737ceb1077211e1712c9da, disabling compactions & flushes 2024-12-04T15:10:06,328 INFO [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:06,328 DEBUG [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:06,328 DEBUG [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. after waiting 0 ms 2024-12-04T15:10:06,328 DEBUG [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:06,328 INFO [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:06,328 DEBUG [RegionOpenAndInit-ns2:test-17333250036801-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0896086d7d737ceb1077211e1712c9da: Waiting for close lock at 1733325006328Disabling compacts and flushes for region at 1733325006328Disabling writes for close at 1733325006328Writing region close event to WAL at 1733325006328Closed at 1733325006328 2024-12-04T15:10:06,330 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17333250036801 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:10:06,331 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733325006330"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733325006330"}]},"ts":"1733325006330"} 2024-12-04T15:10:06,334 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:10:06,335 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17333250036801 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:10:06,335 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17333250036801","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325006335"}]},"ts":"1733325006335"} 2024-12-04T15:10:06,338 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17333250036801, state=ENABLING in hbase:meta 2024-12-04T15:10:06,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17333250036801, region=0896086d7d737ceb1077211e1712c9da, ASSIGN}] 2024-12-04T15:10:06,340 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17333250036801, region=0896086d7d737ceb1077211e1712c9da, ASSIGN 2024-12-04T15:10:06,341 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17333250036801, region=0896086d7d737ceb1077211e1712c9da, ASSIGN; state=OFFLINE, location=ff648bb9ef17,41657,1733324995877; forceNewPlan=false, retain=false 2024-12-04T15:10:06,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=0896086d7d737ceb1077211e1712c9da, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:06,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17333250036801, region=0896086d7d737ceb1077211e1712c9da, ASSIGN because future has completed 2024-12-04T15:10:06,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0896086d7d737ceb1077211e1712c9da, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:06,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:10:06,655 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:06,655 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 0896086d7d737ceb1077211e1712c9da, NAME => 'ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:10:06,656 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:10:06,656 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17333250036801 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,656 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:06,656 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,656 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,658 INFO [StoreOpener-0896086d7d737ceb1077211e1712c9da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,661 INFO [StoreOpener-0896086d7d737ceb1077211e1712c9da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0896086d7d737ceb1077211e1712c9da columnFamilyName f 2024-12-04T15:10:06,661 DEBUG [StoreOpener-0896086d7d737ceb1077211e1712c9da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:06,661 INFO [StoreOpener-0896086d7d737ceb1077211e1712c9da-1 {}] regionserver.HStore(327): Store=0896086d7d737ceb1077211e1712c9da/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:06,662 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,663 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,663 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,664 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,664 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,667 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,670 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:10:06,671 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 0896086d7d737ceb1077211e1712c9da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9555270080, jitterRate=-0.11009612679481506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T15:10:06,671 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:06,672 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 0896086d7d737ceb1077211e1712c9da: Running coprocessor pre-open hook at 1733325006656Writing region info on filesystem at 1733325006656Initializing all the Stores at 1733325006658 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325006658Cleaning up temporary data from old regions at 1733325006664 (+6 ms)Running coprocessor post-open hooks at 1733325006671 (+7 ms)Region opened successfully at 1733325006672 (+1 ms) 2024-12-04T15:10:06,674 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da., pid=13, masterSystemTime=1733325006650 2024-12-04T15:10:06,677 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:06,677 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:06,677 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=0896086d7d737ceb1077211e1712c9da, regionState=OPEN, openSeqNum=2, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:06,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0896086d7d737ceb1077211e1712c9da, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:06,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-12-04T15:10:06,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 0896086d7d737ceb1077211e1712c9da, server=ff648bb9ef17,41657,1733324995877 in 186 msec 2024-12-04T15:10:06,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T15:10:06,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17333250036801, region=0896086d7d737ceb1077211e1712c9da, ASSIGN in 347 msec 2024-12-04T15:10:06,689 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17333250036801 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:10:06,689 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17333250036801","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325006689"}]},"ts":"1733325006689"} 2024-12-04T15:10:06,692 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17333250036801, state=ENABLED in hbase:meta 2024-12-04T15:10:06,693 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17333250036801 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:10:06,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17333250036801 in 1.1910 sec 2024-12-04T15:10:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:10:07,654 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17333250036801 completed 2024-12-04T15:10:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17333250036801', row='row0', locateType=CURRENT is [region=ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da., hostname=ff648bb9ef17,41657,1733324995877, seqNum=2] 2024-12-04T15:10:07,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41657 {}] regionserver.HRegion(8528): writing data to region ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. with WAL disabled. Data may be lost in the event of a crash. 2024-12-04T15:10:07,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17333250036802', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17333250036802 2024-12-04T15:10:07,795 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17333250036802 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:10:07,795 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:07,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17333250036802" procId is: 14 2024-12-04T15:10:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T15:10:07,796 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17333250036802 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:10:07,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741843_1019 (size=390) 2024-12-04T15:10:07,841 INFO [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 505dfcba8179ccbcff79818fd950a242, NAME => 'ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17333250036802', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:10:07,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741844_1020 (size=50) 2024-12-04T15:10:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T15:10:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T15:10:08,250 DEBUG [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:08,251 DEBUG [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(1722): Closing 505dfcba8179ccbcff79818fd950a242, disabling compactions & flushes 2024-12-04T15:10:08,251 INFO [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:10:08,251 DEBUG [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:10:08,251 DEBUG [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. after waiting 0 ms 2024-12-04T15:10:08,251 DEBUG [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:10:08,251 INFO [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:10:08,251 DEBUG [RegionOpenAndInit-ns3:test-17333250036802-pool-0 {}] regionserver.HRegion(1676): Region close journal for 505dfcba8179ccbcff79818fd950a242: Waiting for close lock at 1733325008251Disabling compacts and flushes for region at 1733325008251Disabling writes for close at 1733325008251Writing region close event to WAL at 1733325008251Closed at 1733325008251 2024-12-04T15:10:08,253 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17333250036802 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:10:08,253 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733325008253"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733325008253"}]},"ts":"1733325008253"} 2024-12-04T15:10:08,256 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:10:08,257 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17333250036802 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:10:08,257 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17333250036802","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325008257"}]},"ts":"1733325008257"} 2024-12-04T15:10:08,259 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17333250036802, state=ENABLING in hbase:meta 2024-12-04T15:10:08,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17333250036802, region=505dfcba8179ccbcff79818fd950a242, ASSIGN}] 2024-12-04T15:10:08,261 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17333250036802, region=505dfcba8179ccbcff79818fd950a242, ASSIGN 2024-12-04T15:10:08,262 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17333250036802, region=505dfcba8179ccbcff79818fd950a242, ASSIGN; state=OFFLINE, location=ff648bb9ef17,41657,1733324995877; forceNewPlan=false, retain=false 2024-12-04T15:10:08,413 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=505dfcba8179ccbcff79818fd950a242, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:08,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17333250036802, region=505dfcba8179ccbcff79818fd950a242, ASSIGN because future has completed 2024-12-04T15:10:08,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 505dfcba8179ccbcff79818fd950a242, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T15:10:08,575 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:10:08,575 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => 505dfcba8179ccbcff79818fd950a242, NAME => 'ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:10:08,575 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:10:08,576 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17333250036802 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,576 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:08,576 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,576 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,578 INFO [StoreOpener-505dfcba8179ccbcff79818fd950a242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,580 INFO [StoreOpener-505dfcba8179ccbcff79818fd950a242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 505dfcba8179ccbcff79818fd950a242 columnFamilyName f 2024-12-04T15:10:08,580 DEBUG [StoreOpener-505dfcba8179ccbcff79818fd950a242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:08,581 INFO [StoreOpener-505dfcba8179ccbcff79818fd950a242-1 {}] regionserver.HStore(327): Store=505dfcba8179ccbcff79818fd950a242/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:08,581 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,582 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns3/test-17333250036802/505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,583 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns3/test-17333250036802/505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,584 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,584 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,586 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,589 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns3/test-17333250036802/505dfcba8179ccbcff79818fd950a242/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:10:08,590 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened 505dfcba8179ccbcff79818fd950a242; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10493885280, jitterRate=-0.022680774331092834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T15:10:08,590 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:10:08,591 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for 505dfcba8179ccbcff79818fd950a242: Running coprocessor pre-open hook at 1733325008576Writing region info on filesystem at 1733325008576Initializing all the Stores at 1733325008578 (+2 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325008578Cleaning up temporary data from old regions at 1733325008584 (+6 ms)Running coprocessor post-open hooks at 1733325008590 (+6 ms)Region opened successfully at 1733325008591 (+1 ms) 2024-12-04T15:10:08,593 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242., pid=16, masterSystemTime=1733325008569 2024-12-04T15:10:08,596 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:10:08,596 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:10:08,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=505dfcba8179ccbcff79818fd950a242, regionState=OPEN, openSeqNum=2, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:08,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 505dfcba8179ccbcff79818fd950a242, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:08,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-04T15:10:08,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 505dfcba8179ccbcff79818fd950a242, server=ff648bb9ef17,41657,1733324995877 in 184 msec 2024-12-04T15:10:08,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T15:10:08,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17333250036802, region=505dfcba8179ccbcff79818fd950a242, ASSIGN in 345 msec 2024-12-04T15:10:08,608 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17333250036802 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:10:08,608 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17333250036802","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325008608"}]},"ts":"1733325008608"} 2024-12-04T15:10:08,611 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17333250036802, state=ENABLED in hbase:meta 2024-12-04T15:10:08,612 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17333250036802 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:10:08,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17333250036802 in 821 msec 2024-12-04T15:10:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-12-04T15:10:08,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17333250036802 completed 2024-12-04T15:10:08,934 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17333250036802 get assigned. Timeout = 60000ms 2024-12-04T15:10:08,935 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T15:10:08,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17333250036802 assigned to meta. Checking AM states. 2024-12-04T15:10:08,942 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T15:10:08,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17333250036802 assigned. 2024-12-04T15:10:08,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17333250036803', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17333250036803 2024-12-04T15:10:08,949 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17333250036803 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:10:08,949 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:08,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17333250036803" procId is: 17 2024-12-04T15:10:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:10:08,950 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17333250036803 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:10:08,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741845_1021 (size=390) 2024-12-04T15:10:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:10:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:10:09,362 INFO [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 031ac2bd7ebf036e19eeefd5230f16ab, NAME => 'ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17333250036803', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:10:09,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741846_1022 (size=50) 2024-12-04T15:10:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:10:09,772 DEBUG [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:09,772 DEBUG [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(1722): Closing 031ac2bd7ebf036e19eeefd5230f16ab, disabling compactions & flushes 2024-12-04T15:10:09,772 INFO [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:09,772 DEBUG [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:09,772 DEBUG [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. after waiting 0 ms 2024-12-04T15:10:09,772 DEBUG [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:09,772 INFO [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:09,773 DEBUG [RegionOpenAndInit-ns4:test-17333250036803-pool-0 {}] regionserver.HRegion(1676): Region close journal for 031ac2bd7ebf036e19eeefd5230f16ab: Waiting for close lock at 1733325009772Disabling compacts and flushes for region at 1733325009772Disabling writes for close at 1733325009772Writing region close event to WAL at 1733325009772Closed at 1733325009772 2024-12-04T15:10:09,774 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17333250036803 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:10:09,775 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1733325009774"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733325009774"}]},"ts":"1733325009774"} 2024-12-04T15:10:09,777 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:10:09,778 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17333250036803 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:10:09,779 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17333250036803","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325009778"}]},"ts":"1733325009778"} 2024-12-04T15:10:09,781 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17333250036803, state=ENABLING in hbase:meta 2024-12-04T15:10:09,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17333250036803, region=031ac2bd7ebf036e19eeefd5230f16ab, ASSIGN}] 2024-12-04T15:10:09,783 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17333250036803, region=031ac2bd7ebf036e19eeefd5230f16ab, ASSIGN 2024-12-04T15:10:09,784 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17333250036803, region=031ac2bd7ebf036e19eeefd5230f16ab, ASSIGN; state=OFFLINE, location=ff648bb9ef17,41657,1733324995877; forceNewPlan=false, retain=false 2024-12-04T15:10:09,935 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=031ac2bd7ebf036e19eeefd5230f16ab, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:09,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17333250036803, region=031ac2bd7ebf036e19eeefd5230f16ab, ASSIGN because future has completed 2024-12-04T15:10:09,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 031ac2bd7ebf036e19eeefd5230f16ab, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:10:10,098 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:10,098 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => 031ac2bd7ebf036e19eeefd5230f16ab, NAME => 'ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:10:10,098 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:10:10,099 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17333250036803 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,099 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:10,099 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,099 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,101 INFO [StoreOpener-031ac2bd7ebf036e19eeefd5230f16ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,104 INFO [StoreOpener-031ac2bd7ebf036e19eeefd5230f16ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 031ac2bd7ebf036e19eeefd5230f16ab columnFamilyName f 2024-12-04T15:10:10,104 DEBUG [StoreOpener-031ac2bd7ebf036e19eeefd5230f16ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:10,104 INFO [StoreOpener-031ac2bd7ebf036e19eeefd5230f16ab-1 {}] regionserver.HStore(327): Store=031ac2bd7ebf036e19eeefd5230f16ab/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:10,105 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,106 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns4/test-17333250036803/031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,106 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns4/test-17333250036803/031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,107 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,107 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,109 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,113 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns4/test-17333250036803/031ac2bd7ebf036e19eeefd5230f16ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:10:10,113 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened 031ac2bd7ebf036e19eeefd5230f16ab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10772203840, jitterRate=0.003239661455154419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T15:10:10,113 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:10,114 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for 031ac2bd7ebf036e19eeefd5230f16ab: Running coprocessor pre-open hook at 1733325010099Writing region info on filesystem at 1733325010099Initializing all the Stores at 1733325010100 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325010100Cleaning up temporary data from old regions at 1733325010107 (+7 ms)Running coprocessor post-open hooks at 1733325010113 (+6 ms)Region opened successfully at 1733325010114 (+1 ms) 2024-12-04T15:10:10,116 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab., pid=19, masterSystemTime=1733325010092 2024-12-04T15:10:10,119 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:10,119 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:10,120 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=031ac2bd7ebf036e19eeefd5230f16ab, regionState=OPEN, openSeqNum=2, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:10,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 031ac2bd7ebf036e19eeefd5230f16ab, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:10,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-04T15:10:10,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 031ac2bd7ebf036e19eeefd5230f16ab, server=ff648bb9ef17,41657,1733324995877 in 187 msec 2024-12-04T15:10:10,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T15:10:10,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17333250036803, region=031ac2bd7ebf036e19eeefd5230f16ab, ASSIGN in 348 msec 2024-12-04T15:10:10,133 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17333250036803 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:10:10,134 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17333250036803","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325010133"}]},"ts":"1733325010133"} 2024-12-04T15:10:10,137 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17333250036803, state=ENABLED in hbase:meta 2024-12-04T15:10:10,138 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17333250036803 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:10:10,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17333250036803 in 1.1930 sec 2024-12-04T15:10:10,530 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-04T15:10:10,715 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17333250036803' 2024-12-04T15:10:10,715 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1733325003680' 2024-12-04T15:10:10,716 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17333250036801' 2024-12-04T15:10:10,716 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17333250036802' 2024-12-04T15:10:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:10:11,093 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17333250036803 completed 2024-12-04T15:10:11,093 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17333250036803 get assigned. Timeout = 60000ms 2024-12-04T15:10:11,093 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T15:10:11,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17333250036803 assigned to meta. Checking AM states. 2024-12-04T15:10:11,098 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T15:10:11,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17333250036803 assigned. 2024-12-04T15:10:11,099 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:10:11,099 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:10:11,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:11,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:11,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:11,135 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=179, ProcessCount=12, AvailableMemoryMB=11144 2024-12-04T15:10:11,136 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-12-04T15:10:11,138 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-12-04T15:10:11,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1733325003680 from 'test-1733325003680', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1733325003680', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1733325003680 2024-12-04T15:10:11,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741847_1023 (size=908) 2024-12-04T15:10:11,171 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/.tabledesc/.tableinfo.0000000001.367 2024-12-04T15:10:11,171 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/.tabledesc/.tableinfo.0000000002.908 2024-12-04T15:10:11,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1733325003680}] 2024-12-04T15:10:11,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE}] 2024-12-04T15:10:11,185 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE 2024-12-04T15:10:11,187 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=CLOSING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:11,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE because future has completed 2024-12-04T15:10:11,190 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T15:10:11,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:11,350 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:11,350 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T15:10:11,351 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 44b146fc21bd97567dc283c76bd9e484, disabling compactions & flushes 2024-12-04T15:10:11,351 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:11,351 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:11,351 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. after waiting 0 ms 2024-12-04T15:10:11,351 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:11,355 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 44b146fc21bd97567dc283c76bd9e484 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-12-04T15:10:11,426 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f/09d964a88b174a998307d9ec55b513ad is 37, key is row10/f:q1/1733325005374/Put/seqid=0 2024-12-04T15:10:11,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741848_1024 (size=8413) 2024-12-04T15:10:11,838 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f/09d964a88b174a998307d9ec55b513ad 2024-12-04T15:10:11,901 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f/09d964a88b174a998307d9ec55b513ad as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad 2024-12-04T15:10:11,910 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad, entries=99, sequenceid=103, filesize=8.2 K 2024-12-04T15:10:11,917 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 44b146fc21bd97567dc283c76bd9e484 in 561ms, sequenceid=103, compaction requested=false 2024-12-04T15:10:11,924 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-12-04T15:10:11,927 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:10:11,927 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:11,927 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 44b146fc21bd97567dc283c76bd9e484: Waiting for close lock at 1733325011351Running coprocessor pre-close hooks at 1733325011351Disabling compacts and flushes for region at 1733325011351Disabling writes for close at 1733325011351Obtaining lock to block concurrent updates at 1733325011355 (+4 ms)Preparing flush snapshotting stores in 44b146fc21bd97567dc283c76bd9e484 at 1733325011355Finished memstore snapshotting test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1733325011364 (+9 ms)Flushing stores of test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. at 1733325011366 (+2 ms)Flushing 44b146fc21bd97567dc283c76bd9e484/f: creating writer at 1733325011367 (+1 ms)Flushing 44b146fc21bd97567dc283c76bd9e484/f: appending metadata at 1733325011414 (+47 ms)Flushing 44b146fc21bd97567dc283c76bd9e484/f: closing flushed file at 1733325011417 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48e5daeb: reopening flushed file at 1733325011900 (+483 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 44b146fc21bd97567dc283c76bd9e484 in 561ms, sequenceid=103, compaction requested=false at 1733325011917 (+17 ms)Writing region close event to WAL at 1733325011919 (+2 ms)Running coprocessor post-close hooks at 1733325011924 (+5 ms)Closed at 1733325011927 (+3 ms) 2024-12-04T15:10:11,927 WARN [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 44b146fc21bd97567dc283c76bd9e484 to self. 2024-12-04T15:10:11,931 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:11,931 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=CLOSED 2024-12-04T15:10:11,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:11,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-04T15:10:11,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 in 744 msec 2024-12-04T15:10:11,939 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE; state=CLOSED, location=ff648bb9ef17,41657,1733324995877; forceNewPlan=false, retain=true 2024-12-04T15:10:12,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:12,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE because future has completed 2024-12-04T15:10:12,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:12,251 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:12,251 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 44b146fc21bd97567dc283c76bd9e484, NAME => 'test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:10:12,252 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:10:12,252 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733325003680 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,252 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:12,252 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,252 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,254 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,255 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44b146fc21bd97567dc283c76bd9e484 columnFamilyName f 2024-12-04T15:10:12,255 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:12,265 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad 2024-12-04T15:10:12,265 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(327): Store=44b146fc21bd97567dc283c76bd9e484/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:12,265 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,267 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44b146fc21bd97567dc283c76bd9e484 columnFamilyName f3 2024-12-04T15:10:12,267 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:12,268 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(327): Store=44b146fc21bd97567dc283c76bd9e484/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:12,269 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,276 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44b146fc21bd97567dc283c76bd9e484 columnFamilyName mob 2024-12-04T15:10:12,280 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:12,281 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(327): Store=44b146fc21bd97567dc283c76bd9e484/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:12,281 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,282 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,284 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,285 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,285 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,286 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1733325003680 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T15:10:12,288 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,289 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 44b146fc21bd97567dc283c76bd9e484; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10698376000, jitterRate=-0.003636091947555542}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T15:10:12,289 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:12,290 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 44b146fc21bd97567dc283c76bd9e484: Running coprocessor pre-open hook at 1733325012253Writing region info on filesystem at 1733325012253Initializing all the Stores at 1733325012254 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325012254Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325012254Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325012254Cleaning up temporary data from old regions at 1733325012285 (+31 ms)Running coprocessor post-open hooks at 1733325012289 (+4 ms)Region opened successfully at 1733325012290 (+1 ms) 2024-12-04T15:10:12,291 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., pid=24, masterSystemTime=1733325012246 2024-12-04T15:10:12,293 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:12,294 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:12,294 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=OPEN, openSeqNum=107, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:12,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:12,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-12-04T15:10:12,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 in 205 msec 2024-12-04T15:10:12,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-04T15:10:12,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE in 1.1160 sec 2024-12-04T15:10:12,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-04T15:10:12,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1733325003680 in 1.1290 sec 2024-12-04T15:10:12,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733325003680 in 1.1620 sec 2024-12-04T15:10:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-12-04T15:10:12,314 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1733325003680 completed 2024-12-04T15:10:12,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e30753c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:12,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:12,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:12,316 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:12,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:12,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:12,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1070380d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:12,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:12,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:12,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:12,318 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:12,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aff536b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:12,320 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:12,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:12,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:12,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:12,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733325003680', row='row-test-1733325003680-3-0', locateType=CURRENT is [region=test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., hostname=ff648bb9ef17,41657,1733324995877, seqNum=107] 2024-12-04T15:10:12,371 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:10:12,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is ff648bb9ef17,42357,1733324994229 2024-12-04T15:10:12,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d0ba659 2024-12-04T15:10:12,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:10:12,376 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T15:10:12,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-12-04T15:10:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-12-04T15:10:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T15:10:12,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 10 msec 2024-12-04T15:10:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T15:10:12,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-12-04T15:10:12,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-12-04T15:10:12,535 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:10:12,535 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:12,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-12-04T15:10:12,536 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:10:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T15:10:12,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741849_1025 (size=635) 2024-12-04T15:10:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T15:10:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T15:10:12,949 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e95ffbd37709d518765ec5c865d8f1ed, NAME => 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:10:12,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741850_1026 (size=40) 2024-12-04T15:10:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T15:10:13,360 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:13,360 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing e95ffbd37709d518765ec5c865d8f1ed, disabling compactions & flushes 2024-12-04T15:10:13,360 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:13,360 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:13,360 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. after waiting 0 ms 2024-12-04T15:10:13,360 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:13,360 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:13,361 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for e95ffbd37709d518765ec5c865d8f1ed: Waiting for close lock at 1733325013360Disabling compacts and flushes for region at 1733325013360Disabling writes for close at 1733325013360Writing region close event to WAL at 1733325013360Closed at 1733325013360 2024-12-04T15:10:13,363 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:10:13,363 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1733325013363"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733325013363"}]},"ts":"1733325013363"} 2024-12-04T15:10:13,366 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:10:13,367 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:10:13,367 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325013367"}]},"ts":"1733325013367"} 2024-12-04T15:10:13,370 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-12-04T15:10:13,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN}] 2024-12-04T15:10:13,372 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN 2024-12-04T15:10:13,373 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN; state=OFFLINE, location=ff648bb9ef17,41657,1733324995877; forceNewPlan=false, retain=false 2024-12-04T15:10:13,524 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=e95ffbd37709d518765ec5c865d8f1ed, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:13,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN because future has completed 2024-12-04T15:10:13,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T15:10:13,685 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:13,685 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => e95ffbd37709d518765ec5c865d8f1ed, NAME => 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:10:13,685 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:10:13,686 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,686 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:13,686 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,686 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,688 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,690 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e95ffbd37709d518765ec5c865d8f1ed columnFamilyName meta 2024-12-04T15:10:13,690 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:13,691 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.HStore(327): Store=e95ffbd37709d518765ec5c865d8f1ed/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:13,691 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,693 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e95ffbd37709d518765ec5c865d8f1ed columnFamilyName session 2024-12-04T15:10:13,693 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:13,694 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.HStore(327): Store=e95ffbd37709d518765ec5c865d8f1ed/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:13,694 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,695 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,695 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,696 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,696 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,697 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T15:10:13,700 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,703 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:10:13,704 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened e95ffbd37709d518765ec5c865d8f1ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11063517920, jitterRate=0.030370399355888367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T15:10:13,704 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:13,705 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for e95ffbd37709d518765ec5c865d8f1ed: Running coprocessor pre-open hook at 1733325013686Writing region info on filesystem at 1733325013686Initializing all the Stores at 1733325013687 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325013687Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325013687Cleaning up temporary data from old regions at 1733325013697 (+10 ms)Running coprocessor post-open hooks at 1733325013704 (+7 ms)Region opened successfully at 1733325013705 (+1 ms) 2024-12-04T15:10:13,706 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., pid=28, masterSystemTime=1733325013680 2024-12-04T15:10:13,709 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:13,709 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:13,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=e95ffbd37709d518765ec5c865d8f1ed, regionState=OPEN, openSeqNum=2, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:13,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:13,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-04T15:10:13,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877 in 187 msec 2024-12-04T15:10:13,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-04T15:10:13,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN in 345 msec 2024-12-04T15:10:13,720 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:10:13,721 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325013720"}]},"ts":"1733325013720"} 2024-12-04T15:10:13,723 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-12-04T15:10:13,724 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:10:13,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.1920 sec 2024-12-04T15:10:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-04T15:10:14,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-12-04T15:10:14,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-12-04T15:10:14,686 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-12-04T15:10:14,687 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17333250036802 2024-12-04T15:10:14,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17333250036802 Metrics about Tables on a single HBase RegionServer 2024-12-04T15:10:14,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17333250036801 2024-12-04T15:10:14,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17333250036801 Metrics about Tables on a single HBase RegionServer 2024-12-04T15:10:14,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1733325003680 2024-12-04T15:10:14,689 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1733325003680 Metrics about Tables on a single HBase RegionServer 2024-12-04T15:10:14,690 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17333250036803 2024-12-04T15:10:14,690 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17333250036803 Metrics about Tables on a single HBase RegionServer 2024-12-04T15:10:14,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-12-04T15:10:14,707 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:10:14,707 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:14,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-12-04T15:10:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:10:14,708 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:10:14,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741851_1027 (size=640) 2024-12-04T15:10:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:10:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:10:15,119 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef965fc90d631230a5744cae57d35b0b, NAME => 'backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:10:15,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741852_1028 (size=45) 2024-12-04T15:10:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:10:15,528 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:15,528 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing ef965fc90d631230a5744cae57d35b0b, disabling compactions & flushes 2024-12-04T15:10:15,528 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:15,528 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:15,528 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. after waiting 0 ms 2024-12-04T15:10:15,528 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:15,528 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:15,529 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef965fc90d631230a5744cae57d35b0b: Waiting for close lock at 1733325015528Disabling compacts and flushes for region at 1733325015528Disabling writes for close at 1733325015528Writing region close event to WAL at 1733325015528Closed at 1733325015528 2024-12-04T15:10:15,530 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:10:15,531 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733325015530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733325015530"}]},"ts":"1733325015530"} 2024-12-04T15:10:15,533 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:10:15,534 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:10:15,535 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325015534"}]},"ts":"1733325015534"} 2024-12-04T15:10:15,537 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-12-04T15:10:15,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=ef965fc90d631230a5744cae57d35b0b, ASSIGN}] 2024-12-04T15:10:15,539 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=ef965fc90d631230a5744cae57d35b0b, ASSIGN 2024-12-04T15:10:15,540 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=ef965fc90d631230a5744cae57d35b0b, ASSIGN; state=OFFLINE, location=ff648bb9ef17,41657,1733324995877; forceNewPlan=false, retain=false 2024-12-04T15:10:15,601 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-04T15:10:15,690 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ef965fc90d631230a5744cae57d35b0b, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:15,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=ef965fc90d631230a5744cae57d35b0b, ASSIGN because future has completed 2024-12-04T15:10:15,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef965fc90d631230a5744cae57d35b0b, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:10:15,852 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:15,852 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => ef965fc90d631230a5744cae57d35b0b, NAME => 'backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:10:15,852 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:10:15,853 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,853 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:15,853 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,853 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,855 INFO [StoreOpener-ef965fc90d631230a5744cae57d35b0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,857 INFO [StoreOpener-ef965fc90d631230a5744cae57d35b0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef965fc90d631230a5744cae57d35b0b columnFamilyName meta 2024-12-04T15:10:15,857 DEBUG [StoreOpener-ef965fc90d631230a5744cae57d35b0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:15,857 INFO [StoreOpener-ef965fc90d631230a5744cae57d35b0b-1 {}] regionserver.HStore(327): Store=ef965fc90d631230a5744cae57d35b0b/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:15,857 INFO [StoreOpener-ef965fc90d631230a5744cae57d35b0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,859 INFO [StoreOpener-ef965fc90d631230a5744cae57d35b0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef965fc90d631230a5744cae57d35b0b columnFamilyName session 2024-12-04T15:10:15,859 DEBUG [StoreOpener-ef965fc90d631230a5744cae57d35b0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:15,860 INFO [StoreOpener-ef965fc90d631230a5744cae57d35b0b-1 {}] regionserver.HStore(327): Store=ef965fc90d631230a5744cae57d35b0b/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:15,860 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,861 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system_bulk/ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,862 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system_bulk/ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,863 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,863 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,864 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T15:10:15,866 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,869 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system_bulk/ef965fc90d631230a5744cae57d35b0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:10:15,869 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened ef965fc90d631230a5744cae57d35b0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10861571040, jitterRate=0.011562630534172058}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T15:10:15,870 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:15,870 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for ef965fc90d631230a5744cae57d35b0b: Running coprocessor pre-open hook at 1733325015853Writing region info on filesystem at 1733325015853Initializing all the Stores at 1733325015854 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325015854Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325015854Cleaning up temporary data from old regions at 1733325015863 (+9 ms)Running coprocessor post-open hooks at 1733325015870 (+7 ms)Region opened successfully at 1733325015870 2024-12-04T15:10:15,871 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b., pid=31, masterSystemTime=1733325015846 2024-12-04T15:10:15,874 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:15,874 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:15,875 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ef965fc90d631230a5744cae57d35b0b, regionState=OPEN, openSeqNum=2, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:15,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef965fc90d631230a5744cae57d35b0b, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:15,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-04T15:10:15,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure ef965fc90d631230a5744cae57d35b0b, server=ff648bb9ef17,41657,1733324995877 in 185 msec 2024-12-04T15:10:15,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-04T15:10:15,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=ef965fc90d631230a5744cae57d35b0b, ASSIGN in 345 msec 2024-12-04T15:10:15,886 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:10:15,886 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325015886"}]},"ts":"1733325015886"} 2024-12-04T15:10:15,888 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-12-04T15:10:15,889 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:10:15,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.1870 sec 2024-12-04T15:10:16,718 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-12-04T15:10:16,718 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-12-04T15:10:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:10:16,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-12-04T15:10:16,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:16,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:16,871 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T15:10:16,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., hostname=ff648bb9ef17,41657,1733324995877, seqNum=2] 2024-12-04T15:10:16,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41657 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-12-04T15:10:16,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:10:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325016895 (current time:1733325016895). 2024-12-04T15:10:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-04T15:10:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538686e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:16,897 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:16,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:16,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:16,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2520a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:16,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:16,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:16,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:16,899 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:16,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a662e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:16,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:16,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:16,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:16,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:16,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:10:16,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-12-04T15:10:16,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:10:16,916 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:16,921 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:16,935 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741853_1029 (size=52) 2024-12-04T15:10:16,944 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:16,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed}] 2024-12-04T15:10:16,953 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:10:17,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-12-04T15:10:17,113 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:17,113 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing e95ffbd37709d518765ec5c865d8f1ed 2/2 column families, dataSize=45 B heapSize=632 B 2024-12-04T15:10:17,142 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 is 49, key is activesession:/session:c/1733325016886/Put/seqid=0 2024-12-04T15:10:17,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741854_1030 (size=5049) 2024-12-04T15:10:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:10:17,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:10:17,549 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 2024-12-04T15:10:17,560 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 2024-12-04T15:10:17,567 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85, entries=1, sequenceid=5, filesize=4.9 K 2024-12-04T15:10:17,569 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for e95ffbd37709d518765ec5c865d8f1ed in 456ms, sequenceid=5, compaction requested=false 2024-12-04T15:10:17,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for e95ffbd37709d518765ec5c865d8f1ed: 2024-12-04T15:10:17,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. for snapshot_backup_system completed. 2024-12-04T15:10:17,570 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.' region-info for snapshot=snapshot_backup_system 2024-12-04T15:10:17,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:17,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T15:10:17,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85] hfiles 2024-12-04T15:10:17,578 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 for snapshot=snapshot_backup_system 2024-12-04T15:10:17,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741855_1031 (size=95) 2024-12-04T15:10:17,989 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:17,991 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-04T15:10:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-04T15:10:17,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:17,994 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:17,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-04T15:10:17,998 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:17,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed in 1.0480 sec 2024-12-04T15:10:17,999 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:18,000 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:18,001 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-04T15:10:18,003 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-04T15:10:18,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741856_1032 (size=731) 2024-12-04T15:10:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:10:18,422 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:10:18,435 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:10:18,436 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_backup_system 2024-12-04T15:10:18,438 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:10:18,438 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-12-04T15:10:18,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.5270 sec 2024-12-04T15:10:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:10:19,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T15:10:19,063 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733325012353 started at 1733325019063. 2024-12-04T15:10:19,078 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733325012353 has been started. 2024-12-04T15:10:19,083 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-12-04T15:10:19,088 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-12-04T15:10:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-04T15:10:19,091 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-04T15:10:19,091 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-04T15:10:19,091 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-04T15:10:19,091 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-04T15:10:19,092 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,092 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:19,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T15:10:19,094 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,094 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T15:10:19,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:10:19,095 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,095 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-04T15:10:19,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:19,096 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-04T15:10:19,096 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:19,097 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-04T15:10:19,098 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-04T15:10:19,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-04T15:10:19,099 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-04T15:10:19,099 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-04T15:10:19,099 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-04T15:10:19,099 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-04T15:10:19,099 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'ff648bb9ef17,41657,1733324995877' joining acquired barrier for procedure (rolllog) in zk 2024-12-04T15:10:19,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,101 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:19,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:19,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:19,102 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-04T15:10:19,102 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-04T15:10:19,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:19,103 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:19,103 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:19,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:19,105 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'ff648bb9ef17,41657,1733324995877' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-04T15:10:19,105 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4c6edd64[Count = 0] remaining members to acquire global barrier 2024-12-04T15:10:19,105 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-04T15:10:19,105 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:19,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:19,107 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:19,107 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:19,107 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,107 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-04T15:10:19,107 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-04T15:10:19,108 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-04T15:10:19,108 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,114 INFO [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733324998596 highest: 1733325005004 on ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,115 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.meta:.meta(num 1733324999110) roll requested 2024-12-04T15:10:19,128 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta, exclude list is [], retry=0 2024-12-04T15:10:19,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:19,135 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733324999110.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta 2024-12-04T15:10:19,136 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:19,136 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733324999110.meta is not closed yet, will try archiving it next time 2024-12-04T15:10:19,136 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0:(num 1733324998596) roll requested 2024-12-04T15:10:19,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741834_1010 (size=12055) 2024-12-04T15:10:19,160 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137, exclude list is [], retry=0 2024-12-04T15:10:19,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:19,166 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733324998596 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 2024-12-04T15:10:19,167 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:19,167 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733324998596 is not closed yet, will try archiving it next time 2024-12-04T15:10:19,167 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1:(num 1733325005004) roll requested 2024-12-04T15:10:19,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741833_1009 (size=1034) 2024-12-04T15:10:19,170 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733324998596 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733324998596 2024-12-04T15:10:19,183 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167, exclude list is [], retry=0 2024-12-04T15:10:19,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:19,188 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325005004 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 2024-12-04T15:10:19,189 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:19,189 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325005004 is not closed yet, will try archiving it next time 2024-12-04T15:10:19,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741840_1016 (size=4756) 2024-12-04T15:10:19,191 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325005004 not finished, retry = 0 2024-12-04T15:10:19,195 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 81 2024-12-04T15:10:19,195 INFO [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733325019137 on ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,196 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:10:19,196 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is ff648bb9ef17,42357,1733324994229 2024-12-04T15:10:19,196 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b341894 2024-12-04T15:10:19,196 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:10:19,197 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35597, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-12-04T15:10:19,198 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:19,198 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:19,199 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:19,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58703, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T15:10:19,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:19,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:19,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:36003/backupUT\x00', locateType=CURRENT is [region=backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., hostname=ff648bb9ef17,41657,1733324995877, seqNum=2] 2024-12-04T15:10:19,222 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-04T15:10:19,222 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'ff648bb9ef17,41657,1733324995877' in zk 2024-12-04T15:10:19,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,226 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-04T15:10:19,226 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T15:10:19,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,226 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-04T15:10:19,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:19,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:19,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:19,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:19,227 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:19,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:19,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:19,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'ff648bb9ef17,41657,1733324995877': 2024-12-04T15:10:19,231 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-04T15:10:19,231 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-04T15:10:19,231 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-04T15:10:19,231 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-04T15:10:19,231 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-04T15:10:19,231 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'ff648bb9ef17,41657,1733324995877' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-04T15:10:19,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T15:10:19,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,234 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:19,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:19,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:19,234 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T15:10:19,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:10:19,235 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:19,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:19,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:19,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,237 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:19,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:19,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:19,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T15:10:19,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:19,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T15:10:19,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:19,244 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T15:10:19,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:19,244 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:10:19,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:19,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:19,244 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T15:10:19,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-04T15:10:19,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-04T15:10:19,244 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T15:10:19,244 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:10:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-04T15:10:19,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } 2024-12-04T15:10:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325019259 (current time:1733325019259). 2024-12-04T15:10:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733325019258_default_test-1733325003680 VERSION not specified, setting to 2 2024-12-04T15:10:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e136730, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:19,261 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:19,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:19,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:19,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6111d49f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:19,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:19,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:19,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:19,263 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:19,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@709c3516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:19,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:19,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:19,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:19,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:19,267 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } 2024-12-04T15:10:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-12-04T15:10:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:10:19,271 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:19,272 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:19,275 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:19,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741860_1036 (size=84) 2024-12-04T15:10:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:10:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:10:19,685 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:19,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44b146fc21bd97567dc283c76bd9e484}] 2024-12-04T15:10:19,687 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:19,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-12-04T15:10:19,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:19,840 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 44b146fc21bd97567dc283c76bd9e484 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-12-04T15:10:19,860 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f3/d40dd3248a654e08a7cad97b05eedf91 is 58, key is row-test-1733325003680-3-0/f3:q1/1733325012327/Put/seqid=0 2024-12-04T15:10:19,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741861_1037 (size=5387) 2024-12-04T15:10:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:10:20,267 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f3/d40dd3248a654e08a7cad97b05eedf91 2024-12-04T15:10:20,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741862_1038 (size=4469) 2024-12-04T15:10:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:10:20,708 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/mob/d8fefa4a41ce454ab3752f28ab0e5e1c is 59, key is row-test-1733325003680-3-0/mob:q1/1733325012339/Put/seqid=0 2024-12-04T15:10:20,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741863_1039 (size=5484) 2024-12-04T15:10:21,116 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/mob/d8fefa4a41ce454ab3752f28ab0e5e1c 2024-12-04T15:10:21,125 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f3/d40dd3248a654e08a7cad97b05eedf91 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91 2024-12-04T15:10:21,133 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91, entries=6, sequenceid=121, filesize=5.3 K 2024-12-04T15:10:21,134 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/mob/d8fefa4a41ce454ab3752f28ab0e5e1c as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c 2024-12-04T15:10:21,142 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c, entries=6, sequenceid=121, filesize=5.4 K 2024-12-04T15:10:21,143 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 44b146fc21bd97567dc283c76bd9e484 in 1303ms, sequenceid=121, compaction requested=false 2024-12-04T15:10:21,143 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 44b146fc21bd97567dc283c76bd9e484: 2024-12-04T15:10:21,143 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. for snapshot_1733325019258_default_test-1733325003680 completed. 2024-12-04T15:10:21,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.' region-info for snapshot=snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:21,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:21,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad] hfiles 2024-12-04T15:10:21,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad for snapshot=snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:21,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91] hfiles 2024-12-04T15:10:21,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91 for snapshot=snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:21,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c] hfiles 2024-12-04T15:10:21,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c for snapshot=snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:21,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741864_1040 (size=185) 2024-12-04T15:10:21,355 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-12-04T15:10:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:10:21,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:21,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-04T15:10:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-04T15:10:21,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733325019258_default_test-1733325003680 on region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:21,553 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:21,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-04T15:10:21,558 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:21,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 44b146fc21bd97567dc283c76bd9e484 in 1.8700 sec 2024-12-04T15:10:21,559 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:21,564 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-04T15:10:21,564 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-04T15:10:21,565 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:21,565 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-12-04T15:10:21,565 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:21,566 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-12-04T15:10:21,566 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:21,567 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-12-04T15:10:21,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741865_1041 (size=49) 2024-12-04T15:10:21,575 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:21,575 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:21,577 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:21,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741866_1042 (size=1146) 2024-12-04T15:10:21,997 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:10:22,006 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:10:22,006 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325019258_default_test-1733325003680 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:22,008 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:10:22,008 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-12-04T15:10:22,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1733325019258_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } in 2.7400 sec 2024-12-04T15:10:23,320 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T15:10:23,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:10:23,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1733325003680 completed 2024-12-04T15:10:23,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } 2024-12-04T15:10:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325023443 (current time:1733325023443). 2024-12-04T15:10:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733325023442_ns2_test-17333250036801 VERSION not specified, setting to 2 2024-12-04T15:10:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788f30ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:23,445 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:23,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:23,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:23,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3f8582, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:23,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:23,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:23,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:23,446 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:23,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44740cc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:23,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:23,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:23,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:23,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:23,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:23,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } 2024-12-04T15:10:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-12-04T15:10:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T15:10:23,454 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:23,455 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:23,459 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:23,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741867_1043 (size=86) 2024-12-04T15:10:23,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T15:10:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T15:10:23,866 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:23,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0896086d7d737ceb1077211e1712c9da}] 2024-12-04T15:10:23,867 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:24,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-12-04T15:10:24,020 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:24,020 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 0896086d7d737ceb1077211e1712c9da 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-12-04T15:10:24,040 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/.tmp/f/3e5cab4b92c342bd87890b494aa64314 is 37, key is row10/f:q1/1733325007678/Put/seqid=0 2024-12-04T15:10:24,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741868_1044 (size=8413) 2024-12-04T15:10:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T15:10:24,447 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/.tmp/f/3e5cab4b92c342bd87890b494aa64314 2024-12-04T15:10:24,455 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/.tmp/f/3e5cab4b92c342bd87890b494aa64314 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314 2024-12-04T15:10:24,463 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314, entries=99, sequenceid=103, filesize=8.2 K 2024-12-04T15:10:24,464 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 0896086d7d737ceb1077211e1712c9da in 444ms, sequenceid=103, compaction requested=false 2024-12-04T15:10:24,464 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 0896086d7d737ceb1077211e1712c9da: 2024-12-04T15:10:24,464 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. for snapshot_1733325023442_ns2_test-17333250036801 completed. 2024-12-04T15:10:24,464 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da.' region-info for snapshot=snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:24,464 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:24,464 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314] hfiles 2024-12-04T15:10:24,464 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314 for snapshot=snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:24,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741869_1045 (size=91) 2024-12-04T15:10:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T15:10:24,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-12-04T15:10:24,686 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-12-04T15:10:24,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:24,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-04T15:10:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-04T15:10:24,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733325023442_ns2_test-17333250036801 on region 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:24,872 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:24,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-04T15:10:24,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 0896086d7d737ceb1077211e1712c9da in 1.0070 sec 2024-12-04T15:10:24,875 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:24,876 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:24,877 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:24,877 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:24,878 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:24,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741870_1046 (size=456) 2024-12-04T15:10:25,289 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:10:25,296 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:10:25,297 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325023442_ns2_test-17333250036801 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:25,298 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:10:25,298 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-12-04T15:10:25,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1733325023442_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } in 1.8460 sec 2024-12-04T15:10:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-04T15:10:25,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17333250036801 completed 2024-12-04T15:10:25,603 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1733325012353 2024-12-04T15:10:25,603 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-12-04T15:10:25,605 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-12-04T15:10:25,606 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733325012353_test-1733325003680 2024-12-04T15:10:25,607 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733325019258_default_test-1733325003680 to hdfs://localhost:36003/backupUT/backup_1733325012353/default/test-1733325003680/ 2024-12-04T15:10:25,612 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T15:10:25,646 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36003, inputRoot=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:10:25,646 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1922286205_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36003/backupUT/backup_1733325012353/default/test-1733325003680, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36003/backupUT/backup_1733325012353/default/test-1733325003680/.hbase-snapshot/.tmp/snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:25,649 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T15:10:25,655 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325019258_default_test-1733325003680 to hdfs://localhost:36003/backupUT/backup_1733325012353/default/test-1733325003680/.hbase-snapshot/.tmp/snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:25,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741871_1047 (size=1146) 2024-12-04T15:10:25,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741872_1048 (size=84) 2024-12-04T15:10:26,080 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:26,081 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:26,081 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,092 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-1446655443175122175.jar 2024-12-04T15:10:27,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,093 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,094 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,095 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:27,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:10:27,096 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:10:27,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:10:27,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:10:27,097 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:10:27,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:10:27,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:10:27,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:10:27,098 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:10:27,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:10:27,099 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:10:27,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:10:27,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:10:27,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:10:27,102 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:10:27,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:10:27,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:10:27,103 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:10:27,173 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:10:27,207 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733325019258_default_test-1733325003680' hfile list 2024-12-04T15:10:27,214 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-12-04T15:10:27,444 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local79310544_0001_e6dfaf17-a64a-4cb4-bb7f-0f1e1b399d35/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T15:10:27,445 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local79310544_0001_e6dfaf17-a64a-4cb4-bb7f-0f1e1b399d35/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T15:10:27,658 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-04T15:10:27,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741873_1049 (size=8413) 2024-12-04T15:10:28,167 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad output=hdfs://localhost:36003/backupUT/backup_1733325012353/default/test-1733325003680/archive/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad 2024-12-04T15:10:28,169 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.016M/sec 2024-12-04T15:10:28,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741874_1050 (size=5484) 2024-12-04T15:10:28,616 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c output=hdfs://localhost:36003/backupUT/backup_1733325012353/default/test-1733325003680/archive/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c 2024-12-04T15:10:28,616 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-12-04T15:10:28,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741875_1051 (size=5387) 2024-12-04T15:10:29,083 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91 output=hdfs://localhost:36003/backupUT/backup_1733325012353/default/test-1733325003680/archive/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91 2024-12-04T15:10:29,084 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.011M/sec 2024-12-04T15:10:29,577 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T15:10:29,578 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T15:10:29,587 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:29,587 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733325019258_default_test-1733325003680 finished. 2024-12-04T15:10:29,587 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733325012353_ns2:test-17333250036801 2024-12-04T15:10:29,587 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733325023442_ns2_test-17333250036801 to hdfs://localhost:36003/backupUT/backup_1733325012353/ns2/test-17333250036801/ 2024-12-04T15:10:29,587 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T15:10:29,618 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36003, inputRoot=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:10:29,619 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1922286205_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36003/backupUT/backup_1733325012353/ns2/test-17333250036801, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36003/backupUT/backup_1733325012353/ns2/test-17333250036801/.hbase-snapshot/.tmp/snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:29,621 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T15:10:29,625 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325023442_ns2_test-17333250036801 to hdfs://localhost:36003/backupUT/backup_1733325012353/ns2/test-17333250036801/.hbase-snapshot/.tmp/snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:29,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741876_1052 (size=86) 2024-12-04T15:10:29,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741877_1053 (size=456) 2024-12-04T15:10:30,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:30,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:30,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,036 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-16787930515813866497.jar 2024-12-04T15:10:31,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,037 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,038 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:31,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:10:31,039 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:10:31,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:10:31,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:10:31,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:10:31,040 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:10:31,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:10:31,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:10:31,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:10:31,041 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:10:31,042 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:10:31,042 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:10:31,042 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:10:31,043 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:10:31,043 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:10:31,043 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:10:31,043 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:10:31,043 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:10:31,086 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:10:31,115 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733325023442_ns2_test-17333250036801' hfile list 2024-12-04T15:10:31,117 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-12-04T15:10:31,207 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local930278089_0002_9e21a561-11f1-4370-b776-80b461389231/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T15:10:31,207 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local930278089_0002_9e21a561-11f1-4370-b776-80b461389231/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T15:10:31,285 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-04T15:10:31,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741878_1054 (size=8413) 2024-12-04T15:10:31,355 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314 output=hdfs://localhost:36003/backupUT/backup_1733325012353/ns2/test-17333250036801/archive/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314 2024-12-04T15:10:31,355 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.125M/sec 2024-12-04T15:10:32,253 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T15:10:32,254 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T15:10:32,260 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:32,260 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733325023442_ns2_test-17333250036801 finished. 2024-12-04T15:10:32,261 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1733325003680 2024-12-04T15:10:32,261 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17333250036801 2024-12-04T15:10:32,314 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733325012353 2024-12-04T15:10:32,314 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-12-04T15:10:32,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741879_1055 (size=97) 2024-12-04T15:10:32,724 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:36003/backupUT/backup_1733325012353/.backup.manifest 2024-12-04T15:10:32,726 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733325012353 finished: type=FULL,tablelist=test-1733325003680;ns2:test-17333250036801,targetRootDir=hdfs://localhost:36003/backupUT,startts=1733325019063,completets=1733325032312,bytescopied=0 2024-12-04T15:10:32,726 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-12-04T15:10:32,726 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:32,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1733325019258_default_test-1733325003680" type: DISABLED 2024-12-04T15:10:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733325019258_default_test-1733325003680 2024-12-04T15:10:32,737 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733325019258_default_test-1733325003680 for backup backup_1733325012353 succeeded. 2024-12-04T15:10:32,737 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:32,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1733325023442_ns2_test-17333250036801" type: DISABLED 2024-12-04T15:10:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733325023442_ns2_test-17333250036801 2024-12-04T15:10:32,741 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733325023442_ns2_test-17333250036801 for backup backup_1733325012353 succeeded. 2024-12-04T15:10:32,741 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T15:10:32,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-12-04T15:10:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-04T15:10:32,754 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-04T15:10:32,759 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T15:10:32,774 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733325012353 completed. 2024-12-04T15:10:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:32,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1733325012353', locateType=CURRENT is [region=backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., hostname=ff648bb9ef17,41657,1733324995877, seqNum=2] 2024-12-04T15:10:32,796 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:36003/backupUT/backup_1733325012353/.backup.manifest 2024-12-04T15:10:32,799 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733325012353/.backup.manifest 2024-12-04T15:10:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:32,975 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): writing 99 rows to test-1733325003680 2024-12-04T15:10:32,994 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(119): written 99 rows to test-1733325003680 2024-12-04T15:10:33,144 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(123): written 111 rows to test-1733325003680 to Mob enabled CF 2024-12-04T15:10:33,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17333250036801', row='row-t20', locateType=CURRENT is [region=ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da., hostname=ff648bb9ef17,41657,1733324995877, seqNum=2] 2024-12-04T15:10:33,171 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(136): written 5 rows to ns2:test-17333250036801 2024-12-04T15:10:33,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:33,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] assignment.SplitTableRegionProcedure(210): test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. splitKey isn't explicitly specified, will try to find a best split key from RS ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:33,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad because midkey is the same as first or last row 2024-12-04T15:10:33,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91 because midkey is the same as first or last row 2024-12-04T15:10:33,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c because midkey is the same as first or last row 2024-12-04T15:10:33,183 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.HRegion(2902): Flushing 44b146fc21bd97567dc283c76bd9e484 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-12-04T15:10:33,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f/20099e47536a49fabaae486bca55e64a is 59, key is row-test-1733325003680-1-10/f:q1/1733325032832/Put/seqid=0 2024-12-04T15:10:33,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741880_1056 (size=10796) 2024-12-04T15:10:33,608 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f/20099e47536a49fabaae486bca55e64a 2024-12-04T15:10:33,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484 is 63, key is row-test-1733325003680-3-100/mob:q1/1733325033131/Put/seqid=0 2024-12-04T15:10:33,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741881_1057 (size=5591) 2024-12-04T15:10:34,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:34,037 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/mobdir/data/default/test-1733325003680/8f0eee486f65fbd2ba10e8355ebb380d/mob/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:34,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/mob/40ffe374e1004bcc930d79fa22839b92, store: [table=test-1733325003680 family=mob region=44b146fc21bd97567dc283c76bd9e484] 2024-12-04T15:10:34,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/mob/40ffe374e1004bcc930d79fa22839b92 is 192, key is row-test-1733325003680-3-100/mob:q1/1733325033131/Put/seqid=0 2024-12-04T15:10:34,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741882_1058 (size=13760) 2024-12-04T15:10:34,444 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/mob/40ffe374e1004bcc930d79fa22839b92 2024-12-04T15:10:34,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f/20099e47536a49fabaae486bca55e64a as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/20099e47536a49fabaae486bca55e64a 2024-12-04T15:10:34,460 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/20099e47536a49fabaae486bca55e64a, entries=99, sequenceid=334, filesize=10.5 K 2024-12-04T15:10:34,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/mob/40ffe374e1004bcc930d79fa22839b92 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/40ffe374e1004bcc930d79fa22839b92 2024-12-04T15:10:34,468 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/40ffe374e1004bcc930d79fa22839b92, entries=111, sequenceid=334, filesize=13.4 K 2024-12-04T15:10:34,469 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 44b146fc21bd97567dc283c76bd9e484 in 1286ms, sequenceid=334, compaction requested=false 2024-12-04T15:10:34,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.HRegion(2603): Flush status journal for 44b146fc21bd97567dc283c76bd9e484: 2024-12-04T15:10:34,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/20099e47536a49fabaae486bca55e64a because midkey is the same as first or last row 2024-12-04T15:10:34,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91 because midkey is the same as first or last row 2024-12-04T15:10:34,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/40ffe374e1004bcc930d79fa22839b92 because midkey is the same as first or last row 2024-12-04T15:10:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:10:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.CallRunner(138): callId: 303 service: MasterService methodName: SplitRegion size: 100 connection: 172.17.0.2:50446 deadline: 1733325093178, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-12-04T15:10:34,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1733325003680 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:34,476 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(147): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:34,477 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T15:10:34,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733325003680,, stopping at row=test-1733325003680 ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,488 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733325003680,, stopping at row=test-1733325003680 ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733325003680', row='', locateType=CURRENT is [region=test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., hostname=ff648bb9ef17,41657,1733324995877, seqNum=107] 2024-12-04T15:10:34,499 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(152): split finished in =1328 2024-12-04T15:10:34,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,535 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T15:10:34,538 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733325034499 2024-12-04T15:10:34,545 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733325012353} 2024-12-04T15:10:34,545 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-04T15:10:34,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:34,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:10:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325034584 (current time:1733325034584). 2024-12-04T15:10:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-04T15:10:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456866f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:34,586 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:34,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:34,586 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:34,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c80dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:34,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:34,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:34,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:34,588 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:34,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@774cfbfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:34,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:34,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:34,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:34,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:34,592 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:10:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-12-04T15:10:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:10:34,594 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:34,595 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:34,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:34,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741883_1059 (size=52) 2024-12-04T15:10:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:10:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:10:35,004 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:35,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed}] 2024-12-04T15:10:35,005 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:35,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-12-04T15:10:35,156 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:35,156 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing e95ffbd37709d518765ec5c865d8f1ed 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-12-04T15:10:35,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/7878fc449f4a44fe851a48d3494268ef is 158, key is trslm:hdfs://localhost:36003/backupUT\x00ns2:test-17333250036801/meta:log-roll-map/1733325032302/Put/seqid=0 2024-12-04T15:10:35,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741884_1060 (size=5914) 2024-12-04T15:10:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:10:35,578 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/7878fc449f4a44fe851a48d3494268ef 2024-12-04T15:10:35,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/00a6b3bf73454805ba3274fdb0479620 is 433, key is session:backup_1733325012353/session:context/1733325032758/Put/seqid=0 2024-12-04T15:10:35,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741885_1061 (size=5535) 2024-12-04T15:10:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:10:35,994 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/00a6b3bf73454805ba3274fdb0479620 2024-12-04T15:10:36,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/7878fc449f4a44fe851a48d3494268ef as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef 2024-12-04T15:10:36,007 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef, entries=6, sequenceid=17, filesize=5.8 K 2024-12-04T15:10:36,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/00a6b3bf73454805ba3274fdb0479620 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620 2024-12-04T15:10:36,013 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620, entries=2, sequenceid=17, filesize=5.4 K 2024-12-04T15:10:36,014 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for e95ffbd37709d518765ec5c865d8f1ed in 858ms, sequenceid=17, compaction requested=false 2024-12-04T15:10:36,014 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for e95ffbd37709d518765ec5c865d8f1ed: 2024-12-04T15:10:36,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. for snapshot_backup_system completed. 2024-12-04T15:10:36,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.' region-info for snapshot=snapshot_backup_system 2024-12-04T15:10:36,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:36,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef] hfiles 2024-12-04T15:10:36,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef for snapshot=snapshot_backup_system 2024-12-04T15:10:36,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620] hfiles 2024-12-04T15:10:36,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 for snapshot=snapshot_backup_system 2024-12-04T15:10:36,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620 for snapshot=snapshot_backup_system 2024-12-04T15:10:36,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741886_1062 (size=173) 2024-12-04T15:10:36,421 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:36,421 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-12-04T15:10:36,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-12-04T15:10:36,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:36,422 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:36,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-04T15:10:36,425 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:36,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed in 1.4190 sec 2024-12-04T15:10:36,426 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:36,426 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:36,426 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-04T15:10:36,427 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-04T15:10:36,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741887_1063 (size=810) 2024-12-04T15:10:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:10:36,839 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:10:36,846 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:10:36,847 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_backup_system 2024-12-04T15:10:36,849 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:10:36,849 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-12-04T15:10:36,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2570 sec 2024-12-04T15:10:38,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:10:38,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T15:10:38,763 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733325034499 started at 1733325038763. 2024-12-04T15:10:38,765 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733325034499 has been started. 2024-12-04T15:10:38,770 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [ns2:test-17333250036801, test-1733325003680] 2024-12-04T15:10:38,774 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1733325005004for backupID backup_1733325034499 2024-12-04T15:10:38,774 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-12-04T15:10:38,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-12-04T15:10:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-04T15:10:38,775 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-04T15:10:38,775 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-04T15:10:38,775 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-04T15:10:38,775 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-04T15:10:38,775 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,776 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:38,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T15:10:38,777 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,777 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T15:10:38,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:10:38,778 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,778 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-04T15:10:38,778 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:38,778 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-04T15:10:38,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:38,779 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-04T15:10:38,779 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-04T15:10:38,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-04T15:10:38,779 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-04T15:10:38,779 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-04T15:10:38,779 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-04T15:10:38,779 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-04T15:10:38,779 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'ff648bb9ef17,41657,1733324995877' joining acquired barrier for procedure (rolllog) in zk 2024-12-04T15:10:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,781 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:38,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:38,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:38,782 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-04T15:10:38,782 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-04T15:10:38,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:38,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:38,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:38,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:38,784 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'ff648bb9ef17,41657,1733324995877' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-04T15:10:38,784 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@49512822[Count = 0] remaining members to acquire global barrier 2024-12-04T15:10:38,784 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-04T15:10:38,784 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:38,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:38,786 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:38,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:38,786 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-04T15:10:38,786 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-04T15:10:38,786 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,786 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,786 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-04T15:10:38,787 INFO [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733325019137 highest: 1733325019167 on ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,787 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.meta:.meta(num 1733325019115) roll requested 2024-12-04T15:10:38,801 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325038787.meta, exclude list is [], retry=0 2024-12-04T15:10:38,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:38,805 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325038787.meta 2024-12-04T15:10:38,806 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:38,806 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta is not closed yet, will try archiving it next time 2024-12-04T15:10:38,806 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0:(num 1733325019137) roll requested 2024-12-04T15:10:38,807 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta not finished, retry = 0 2024-12-04T15:10:38,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741857_1033 (size=93) 2024-12-04T15:10:38,820 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325038806, exclude list is [], retry=0 2024-12-04T15:10:38,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:38,824 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325038806 2024-12-04T15:10:38,824 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:38,824 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 is not closed yet, will try archiving it next time 2024-12-04T15:10:38,824 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1:(num 1733325019167) roll requested 2024-12-04T15:10:38,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741858_1034 (size=1198) 2024-12-04T15:10:38,826 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 not finished, retry = 0 2024-12-04T15:10:38,838 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325038824, exclude list is [], retry=0 2024-12-04T15:10:38,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:38,843 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325038824 2024-12-04T15:10:38,843 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:38,843 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 is not closed yet, will try archiving it next time 2024-12-04T15:10:38,843 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325005004 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325005004 2024-12-04T15:10:38,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741859_1035 (size=30490) 2024-12-04T15:10:38,847 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-04T15:10:38,847 INFO [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733325038806 on ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:38,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:38,864 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-04T15:10:38,864 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'ff648bb9ef17,41657,1733324995877' in zk 2024-12-04T15:10:38,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,868 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-04T15:10:38,868 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T15:10:38,868 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-04T15:10:38,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:38,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:38,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:38,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:38,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:38,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:38,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:38,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'ff648bb9ef17,41657,1733324995877': 2024-12-04T15:10:38,872 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'ff648bb9ef17,41657,1733324995877' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-04T15:10:38,872 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-04T15:10:38,872 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-04T15:10:38,872 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-04T15:10:38,872 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-04T15:10:38,872 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-04T15:10:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T15:10:38,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,874 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:38,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:38,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:38,875 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T15:10:38,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:10:38,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:38,875 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:38,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:38,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,877 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:38,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:38,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:38,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T15:10:38,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:38,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:38,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:38,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:38,880 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T15:10:38,880 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:10:38,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:38,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T15:10:38,881 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T15:10:38,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-04T15:10:38,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-04T15:10:38,881 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T15:10:38,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:10:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-04T15:10:38,894 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {ff648bb9ef17:41657=1733325005004} newestTimestamps: {ff648bb9ef17:41657=1733325019167} 2024-12-04T15:10:38,897 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: ff648bb9ef17%2C41657%2C1733324995877.meta.1733324999110.meta 2024-12-04T15:10:38,897 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta 2024-12-04T15:10:38,897 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: ff648bb9ef17%2C41657%2C1733324995877.meta.1733325038787.meta 2024-12-04T15:10:38,897 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 2024-12-04T15:10:38,897 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325038806 2024-12-04T15:10:38,897 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 2024-12-04T15:10:38,897 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325038824 2024-12-04T15:10:38,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741891_1067 (size=908) 2024-12-04T15:10:38,908 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta 2024-12-04T15:10:39,309 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1733325003680 target: hdfs://localhost:36003/backupUT/backup_1733325034499/default/test-1733325003680 descriptor: 'test-1733325003680', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:39,309 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T15:10:39,312 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1733325003680 2024-12-04T15:10:39,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741892_1068 (size=53) 2024-12-04T15:10:39,718 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1733325003680 2024-12-04T15:10:39,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741893_1069 (size=364) 2024-12-04T15:10:39,979 DEBUG [master/ff648bb9ef17:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733325003680 2024-12-04T15:10:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741894_1070 (size=950) 2024-12-04T15:10:40,130 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17333250036801 target: hdfs://localhost:36003/backupUT/backup_1733325034499/ns2/test-17333250036801 descriptor: 'ns2:test-17333250036801', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:40,130 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T15:10:40,132 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17333250036801 2024-12-04T15:10:40,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741895_1071 (size=50) 2024-12-04T15:10:40,392 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/.tabledesc/.tableinfo.0000000002.908 2024-12-04T15:10:40,392 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/.tabledesc/.tableinfo.0000000003.950 2024-12-04T15:10:40,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733325003680 in 417 msec 2024-12-04T15:10:40,538 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17333250036801 2024-12-04T15:10:40,548 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:36003/backupUT/.tmp/backup_1733325034499 from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137;hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 2024-12-04T15:10:40,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fd1d969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:40,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:40,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:40,552 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:40,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:40,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:40,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b6d75a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:40,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:40,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:40,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:40,553 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:40,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ec60d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:40,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:10:40,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is ff648bb9ef17,42357,1733324994229 2024-12-04T15:10:40,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52978775 2024-12-04T15:10:40,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:10:40,556 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T15:10:40,562 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T15:10:40,562 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17333250036801 2024-12-04T15:10:40,562 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17333250036801,, stopping at row=ns2:test-17333250036801 ,, for max=2147483647 with caching=100 2024-12-04T15:10:40,563 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:40,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:40,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:40,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:40,567 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17333250036801: ns2:test-17333250036801; 2024-12-04T15:10:40,567 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1733325003680 2024-12-04T15:10:40,568 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733325003680,, stopping at row=test-1733325003680 ,, for max=2147483647 with caching=100 2024-12-04T15:10:40,570 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1733325003680: test-1733325003680; 2024-12-04T15:10:40,570 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-12-04T15:10:40,570 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/hbase-staging/partitions_af20e891-e773-4de4-a420-86c7e3d806d8 2024-12-04T15:10:40,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741896_1072 (size=192) 2024-12-04T15:10:40,995 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:40,995 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:40,995 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:41,999 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-5522571952932275161.jar 2024-12-04T15:10:42,000 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,000 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,000 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,001 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,001 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,001 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,001 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,002 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,002 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,002 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:10:42,002 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:10:42,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:10:42,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:10:42,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:10:42,003 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:10:42,004 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:10:42,004 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:10:42,004 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:10:42,004 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:10:42,005 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:10:42,006 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,006 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,007 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:10:42,008 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:10:42,008 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: ns2:test-17333250036801,default:test-1733325003680 2024-12-04T15:10:42,008 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:10:42,008 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:157) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:10:42,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:42,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:42,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:42,009 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:10:42,998 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-2908126393856731969.jar 2024-12-04T15:10:43,012 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:10:43,039 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 for WAL files 2024-12-04T15:10:43,041 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 2024-12-04T15:10:43,041 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 for WAL files 2024-12-04T15:10:43,042 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 2024-12-04T15:10:43,146 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local875446097_0003_392a88fe-acfa-4dee-b8c8-880b703c4fc6/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T15:10:43,146 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local875446097_0003_392a88fe-acfa-4dee-b8c8-880b703c4fc6/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T15:10:43,248 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 for hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 (-9223372036854775808:9223372036854775807) length:30490 2024-12-04T15:10:43,267 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-04T15:10:43,267 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-04T15:10:43,317 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 for hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 (-9223372036854775808:9223372036854775807) length:1198 2024-12-04T15:10:43,321 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-12-04T15:10:43,321 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-12-04T15:10:43,407 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d407d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,407 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:43,407 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:43,408 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:43,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:43,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:43,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7f9dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:43,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:43,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,410 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:43,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e85580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,411 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:43,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:43,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:43,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:43,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17333250036801', row='row-t20', locateType=CURRENT is [region=ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da., hostname=ff648bb9ef17,41657,1733324995877, seqNum=2] 2024-12-04T15:10:43,415 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-04T15:10:43,415 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:10:43,415 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,416 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,416 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:43,416 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-12-04T15:10:43,416 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: ff648bb9ef17 2024-12-04T15:10:43,421 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/backupUT/.tmp/backup_1733325034499/_temporary/0/_temporary/attempt_local875446097_0003_r_000000_0/ns2/test-17333250036801/f/a007d1a59c2e431ea29895b2bfe9be6c is 38, key is row-t20/f:q1/1733325033159/Put/seqid=0 2024-12-04T15:10:43,423 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [ff648bb9ef17:41657] Specified favored nodes: [ff648bb9ef17:41657] 2024-12-04T15:10:43,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741897_1073 (size=5274) 2024-12-04T15:10:43,856 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c7580c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,857 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:43,857 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:43,858 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:43,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:43,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:43,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4f4975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:43,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:43,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,859 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:43,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c763f5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,860 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:43,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:43,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:43,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:43,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733325003680', row='row-test-1733325003680-1-0', locateType=CURRENT is [region=test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., hostname=ff648bb9ef17,41657,1733324995877, seqNum=107] 2024-12-04T15:10:43,864 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-04T15:10:43,865 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:10:43,865 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,865 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:43,865 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733325003680-1-0] 2024-12-04T15:10:43,865 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: ff648bb9ef17 2024-12-04T15:10:43,875 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6d006d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,875 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:43,875 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:43,876 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:43,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:43,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:43,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0782a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:43,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:43,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:43,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be67049, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:43,878 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:43,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:43,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:43,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:43,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1733325003680', row='row-test-1733325003680-3-0', locateType=CURRENT is [region=test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., hostname=ff648bb9ef17,41657,1733324995877, seqNum=107] 2024-12-04T15:10:43,882 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-12-04T15:10:43,882 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:10:43,882 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,882 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:43,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:43,882 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1733325003680-3-0] 2024-12-04T15:10:43,882 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: ff648bb9ef17 2024-12-04T15:10:43,890 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/backupUT/.tmp/backup_1733325034499/_temporary/0/_temporary/attempt_local875446097_0003_r_000001_0/default/test-1733325003680/f/ad2fe40906904b66a6ee1515199e9c93 is 59, key is row-test-1733325003680-1-10/f:q1/1733325032832/Put/seqid=0 2024-12-04T15:10:43,891 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [ff648bb9ef17:41657] Specified favored nodes: [ff648bb9ef17:41657] 2024-12-04T15:10:43,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741898_1074 (size=10769) 2024-12-04T15:10:44,295 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/backupUT/.tmp/backup_1733325034499/_temporary/0/_temporary/attempt_local875446097_0003_r_000001_0/default/test-1733325003680/mob/7dfc8ef8563d47599d86773db446309e is 63, key is row-test-1733325003680-3-100/mob:q1/1733325033131/Put/seqid=0 2024-12-04T15:10:44,297 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [ff648bb9ef17:41657] Specified favored nodes: [ff648bb9ef17:41657] 2024-12-04T15:10:44,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741899_1075 (size=11810) 2024-12-04T15:10:45,192 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:36003/backupUT 2024-12-04T15:10:45,192 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1733325034499 2024-12-04T15:10:45,192 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-12-04T15:10:45,212 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:36003/backupUT/.tmp/backup_1733325034499, hdfs://localhost:36003/backupUT] 2024-12-04T15:10:45,284 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:10:45,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741900_1076 (size=5274) 2024-12-04T15:10:45,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741901_1077 (size=10769) 2024-12-04T15:10:46,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741902_1078 (size=11810) 2024-12-04T15:10:46,899 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-12-04T15:10:46,901 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1733325034499 2024-12-04T15:10:46,902 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27853 bytes copied." 2024-12-04T15:10:46,902 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local100221933_0004 completed: true true 2024-12-04T15:10:46,903 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3831857 FILE: Number of bytes written=6202690 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231519 HDFS: Number of bytes written=2433104 HDFS: Number of read operations=646 HDFS: Number of large read operations=0 HDFS: Number of write operations=251 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=340 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27853 Bytes Copied=27853 Bytes Expected=27853 Files Copied=4 DIR_COPY=8 2024-12-04T15:10:46,903 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:36003/backupUT/.tmp/backup_1733325034499 to hdfs://localhost:36003/backupUT finished. 2024-12-04T15:10:46,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:test-1733325003680:', locateType=CURRENT is [region=backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b., hostname=ff648bb9ef17,41657,1733324995877, seqNum=2] 2024-12-04T15:10:46,918 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733325034499 2024-12-04T15:10:46,921 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733325012353} 2024-12-04T15:10:46,921 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-04T15:10:46,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741903_1079 (size=317) 2024-12-04T15:10:47,327 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:36003/backupUT/backup_1733325034499/.backup.manifest 2024-12-04T15:10:47,327 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733325034499 finished: type=INCREMENTAL,tablelist=test-1733325003680;ns2:test-17333250036801,targetRootDir=hdfs://localhost:36003/backupUT,startts=1733325038763,completets=1733325046918,bytescopied=0 2024-12-04T15:10:47,327 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T15:10:47,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-12-04T15:10:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-04T15:10:47,335 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-04T15:10:47,338 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T15:10:47,341 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733325034499 completed. 2024-12-04T15:10:47,342 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-12-04T15:10:47,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:47,358 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:36003/backupUT/backup_1733325034499/.backup.manifest 2024-12-04T15:10:47,360 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733325034499/.backup.manifest 2024-12-04T15:10:47,361 DEBUG [Time-limited test {}] impl.BackupManifest(419): Loading manifest from: hdfs://localhost:36003/backupUT/backup_1733325034499/.backup.manifest 2024-12-04T15:10:47,363 DEBUG [Time-limited test {}] impl.BackupManifest(449): Loaded manifest instance from manifest file: /backupUT/backup_1733325034499/.backup.manifest 2024-12-04T15:10:47,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1733325003680 from 'test-1733325003680', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1733325003680', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:10:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1733325003680 2024-12-04T15:10:47,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741904_1080 (size=908) 2024-12-04T15:10:47,778 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/.tabledesc/.tableinfo.0000000003.950 2024-12-04T15:10:47,778 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/.tabledesc/.tableinfo.0000000004.908 2024-12-04T15:10:47,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1733325003680}] 2024-12-04T15:10:47,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE}] 2024-12-04T15:10:47,784 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE 2024-12-04T15:10:47,785 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=CLOSING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:47,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE because future has completed 2024-12-04T15:10:47,787 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T15:10:47,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:47,940 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:47,940 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T15:10:47,940 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 44b146fc21bd97567dc283c76bd9e484, disabling compactions & flushes 2024-12-04T15:10:47,940 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:47,941 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:47,941 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. after waiting 0 ms 2024-12-04T15:10:47,941 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:47,946 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-12-04T15:10:47,947 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:10:47,947 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:47,947 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 44b146fc21bd97567dc283c76bd9e484: Waiting for close lock at 1733325047940Running coprocessor pre-close hooks at 1733325047940Disabling compacts and flushes for region at 1733325047940Disabling writes for close at 1733325047941 (+1 ms)Writing region close event to WAL at 1733325047942 (+1 ms)Running coprocessor post-close hooks at 1733325047947 (+5 ms)Closed at 1733325047947 2024-12-04T15:10:47,947 WARN [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 44b146fc21bd97567dc283c76bd9e484 to self. 2024-12-04T15:10:47,949 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:47,950 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=CLOSED 2024-12-04T15:10:47,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:47,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-04T15:10:47,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 in 165 msec 2024-12-04T15:10:47,955 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE; state=CLOSED, location=ff648bb9ef17,41657,1733324995877; forceNewPlan=false, retain=true 2024-12-04T15:10:48,105 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:48,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE because future has completed 2024-12-04T15:10:48,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:10:48,262 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:48,263 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 44b146fc21bd97567dc283c76bd9e484, NAME => 'test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:10:48,263 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:10:48,263 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1733325003680 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,264 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:10:48,264 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,264 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,265 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,266 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44b146fc21bd97567dc283c76bd9e484 columnFamilyName f 2024-12-04T15:10:48,266 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:48,273 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad 2024-12-04T15:10:48,278 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/20099e47536a49fabaae486bca55e64a 2024-12-04T15:10:48,278 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(327): Store=44b146fc21bd97567dc283c76bd9e484/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:48,278 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,279 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44b146fc21bd97567dc283c76bd9e484 columnFamilyName f2 2024-12-04T15:10:48,280 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:48,280 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(327): Store=44b146fc21bd97567dc283c76bd9e484/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:48,280 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,281 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44b146fc21bd97567dc283c76bd9e484 columnFamilyName mob 2024-12-04T15:10:48,282 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:48,288 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/40ffe374e1004bcc930d79fa22839b92 2024-12-04T15:10:48,292 DEBUG [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c 2024-12-04T15:10:48,292 INFO [StoreOpener-44b146fc21bd97567dc283c76bd9e484-1 {}] regionserver.HStore(327): Store=44b146fc21bd97567dc283c76bd9e484/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:10:48,293 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,293 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,294 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,295 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,295 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,296 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1733325003680 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T15:10:48,297 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,298 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 44b146fc21bd97567dc283c76bd9e484; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10906597440, jitterRate=0.015756040811538696}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T15:10:48,298 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:48,300 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 44b146fc21bd97567dc283c76bd9e484: Running coprocessor pre-open hook at 1733325048264Writing region info on filesystem at 1733325048264Initializing all the Stores at 1733325048265 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325048265Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325048265Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325048265Cleaning up temporary data from old regions at 1733325048295 (+30 ms)Running coprocessor post-open hooks at 1733325048298 (+3 ms)Region opened successfully at 1733325048300 (+2 ms) 2024-12-04T15:10:48,301 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., pid=46, masterSystemTime=1733325048259 2024-12-04T15:10:48,303 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:48,303 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:48,304 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=44b146fc21bd97567dc283c76bd9e484, regionState=OPEN, openSeqNum=338, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:48,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:10:48,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-12-04T15:10:48,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 44b146fc21bd97567dc283c76bd9e484, server=ff648bb9ef17,41657,1733324995877 in 199 msec 2024-12-04T15:10:48,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-12-04T15:10:48,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1733325003680, region=44b146fc21bd97567dc283c76bd9e484, REOPEN/MOVE in 526 msec 2024-12-04T15:10:48,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-04T15:10:48,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1733325003680 in 531 msec 2024-12-04T15:10:48,315 DEBUG [PEWorker-1 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1733325003680 2024-12-04T15:10:48,322 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f3/d40dd3248a654e08a7cad97b05eedf91 2024-12-04T15:10:48,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1733325003680 in 958 msec 2024-12-04T15:10:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T15:10:48,326 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1733325003680 completed 2024-12-04T15:10:48,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,359 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T15:10:48,363 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733325048326 2024-12-04T15:10:48,365 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1733325034499} 2024-12-04T15:10:48,365 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733325012353} 2024-12-04T15:10:48,365 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-12-04T15:10:48,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,406 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1733325003680 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1733325003680 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:175) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:174) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:10:48,408 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1733325048326,startts=0,failedts=1733325048406,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1733325003680 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-12-04T15:10:48,408 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-12-04T15:10:48,411 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-12-04T15:10:48,411 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T15:10:48,412 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-12-04T15:10:48,412 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-12-04T15:10:48,414 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T15:10:48,417 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1733325048326 failed. 2024-12-04T15:10:48,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:48,435 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1733325003680, ns2:test-17333250036801, ns3:test-17333250036802, ns4:test-17333250036803] 2024-12-04T15:10:48,435 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T15:10:48,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:10:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325048440 (current time:1733325048440). 2024-12-04T15:10:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-04T15:10:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27016722, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:48,441 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:48,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:48,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:48,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35052ddd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:48,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:48,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:48,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:48,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:48,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e98011, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:48,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:48,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:48,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:48,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:48,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:10:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-12-04T15:10:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:10:48,449 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:48,450 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:48,453 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:48,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741905_1081 (size=52) 2024-12-04T15:10:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:10:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:10:48,859 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:48,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed}] 2024-12-04T15:10:48,860 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:49,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-12-04T15:10:49,013 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:49,013 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing e95ffbd37709d518765ec5c865d8f1ed 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-12-04T15:10:49,018 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/b3329fbe4f474becb485a64cbd0f94b0 is 158, key is trslm:hdfs://localhost:36003/backupUT\x00ns2:test-17333250036801/meta:log-roll-map/1733325046908/Put/seqid=0 2024-12-04T15:10:49,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741906_1082 (size=5745) 2024-12-04T15:10:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:10:49,377 DEBUG [master/ff648bb9ef17:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733325003680 2024-12-04T15:10:49,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741907_1083 (size=950) 2024-12-04T15:10:49,423 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/b3329fbe4f474becb485a64cbd0f94b0 2024-12-04T15:10:49,432 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/f3a833403bfc4f00b96e0f5a1f10406e is 1245, key is session:backup_1733325048326/session:context/1733325048413/Put/seqid=0 2024-12-04T15:10:49,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741908_1084 (size=6682) 2024-12-04T15:10:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:10:49,789 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/.tabledesc/.tableinfo.0000000004.908 2024-12-04T15:10:49,789 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/.tabledesc/.tableinfo.0000000005.950 2024-12-04T15:10:49,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1733325003680 in 413 msec 2024-12-04T15:10:49,837 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/f3a833403bfc4f00b96e0f5a1f10406e 2024-12-04T15:10:49,844 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/b3329fbe4f474becb485a64cbd0f94b0 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0 2024-12-04T15:10:49,849 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0, entries=4, sequenceid=31, filesize=5.6 K 2024-12-04T15:10:49,850 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/f3a833403bfc4f00b96e0f5a1f10406e as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e 2024-12-04T15:10:49,855 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e, entries=3, sequenceid=31, filesize=6.5 K 2024-12-04T15:10:49,856 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for e95ffbd37709d518765ec5c865d8f1ed in 843ms, sequenceid=31, compaction requested=true 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for e95ffbd37709d518765ec5c865d8f1ed: 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. for snapshot_backup_system completed. 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.' region-info for snapshot=snapshot_backup_system 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0] hfiles 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef for snapshot=snapshot_backup_system 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0 for snapshot=snapshot_backup_system 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e] hfiles 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 for snapshot=snapshot_backup_system 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620 for snapshot=snapshot_backup_system 2024-12-04T15:10:49,856 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e for snapshot=snapshot_backup_system 2024-12-04T15:10:49,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741909_1085 (size=251) 2024-12-04T15:10:50,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:10:50,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-04T15:10:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-12-04T15:10:50,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:50,263 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:10:50,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-12-04T15:10:50,266 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:50,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed in 1.4050 sec 2024-12-04T15:10:50,267 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:50,267 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:50,267 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-04T15:10:50,268 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-04T15:10:50,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741910_1086 (size=888) 2024-12-04T15:10:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:10:50,678 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:10:50,686 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:10:50,687 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_backup_system 2024-12-04T15:10:50,688 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:10:50,688 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-12-04T15:10:50,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2410 sec 2024-12-04T15:10:51,656 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 0896086d7d737ceb1077211e1712c9da, had cached 0 bytes from a total of 8413 2024-12-04T15:10:52,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:10:52,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T15:10:52,613 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733325048418 started at 1733325052613. 2024-12-04T15:10:52,615 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733325048418 has been started. 2024-12-04T15:10:52,619 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-12-04T15:10:52,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-12-04T15:10:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-04T15:10:52,620 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-04T15:10:52,620 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-04T15:10:52,620 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-04T15:10:52,620 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-04T15:10:52,621 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,621 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:52,622 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T15:10:52,623 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T15:10:52,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:10:52,623 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,623 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-04T15:10:52,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:52,624 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-04T15:10:52,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:52,624 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-04T15:10:52,625 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-04T15:10:52,625 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-04T15:10:52,625 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-04T15:10:52,625 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-04T15:10:52,625 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-04T15:10:52,625 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-04T15:10:52,625 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'ff648bb9ef17,41657,1733324995877' joining acquired barrier for procedure (rolllog) in zk 2024-12-04T15:10:52,627 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:52,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,627 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,627 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:52,627 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:52,627 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-04T15:10:52,627 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-04T15:10:52,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:52,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:52,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:52,629 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,629 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:52,629 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'ff648bb9ef17,41657,1733324995877' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-04T15:10:52,629 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@25dae2a9[Count = 0] remaining members to acquire global barrier 2024-12-04T15:10:52,629 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-04T15:10:52,629 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:52,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:52,631 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:52,631 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:52,631 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-04T15:10:52,631 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,631 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-04T15:10:52,631 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-04T15:10:52,631 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,631 INFO [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733325038806 highest: 1733325038824 on ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,631 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.meta:.meta(num 1733325038787) roll requested 2024-12-04T15:10:52,645 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325052631.meta, exclude list is [], retry=0 2024-12-04T15:10:52,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:52,650 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325038787.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325052631.meta 2024-12-04T15:10:52,650 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:52,650 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325038787.meta is not closed yet, will try archiving it next time 2024-12-04T15:10:52,650 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0:(num 1733325038806) roll requested 2024-12-04T15:10:52,651 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325038787.meta not finished, retry = 0 2024-12-04T15:10:52,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741888_1064 (size=1903) 2024-12-04T15:10:52,664 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650, exclude list is [], retry=0 2024-12-04T15:10:52,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:52,668 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325038806 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 2024-12-04T15:10:52,668 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:52,668 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325038806 is not closed yet, will try archiving it next time 2024-12-04T15:10:52,668 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1:(num 1733325038824) roll requested 2024-12-04T15:10:52,669 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325038806 not finished, retry = 0 2024-12-04T15:10:52,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741889_1065 (size=93) 2024-12-04T15:10:52,681 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325052668, exclude list is [], retry=0 2024-12-04T15:10:52,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:10:52,684 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325038824 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325052668 2024-12-04T15:10:52,685 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:10:52,685 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325038824 is not closed yet, will try archiving it next time 2024-12-04T15:10:52,685 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325019167 2024-12-04T15:10:52,686 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325038824 not finished, retry = 0 2024-12-04T15:10:52,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741890_1066 (size=4927) 2024-12-04T15:10:52,692 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-12-04T15:10:52,692 INFO [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733325052650 on ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:10:52,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:10:52,707 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-04T15:10:52,707 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'ff648bb9ef17,41657,1733324995877' in zk 2024-12-04T15:10:52,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,710 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-04T15:10:52,710 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T15:10:52,710 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-04T15:10:52,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:52,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:52,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:52,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:52,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:52,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,713 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:52,713 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:52,713 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,714 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'ff648bb9ef17,41657,1733324995877': 2024-12-04T15:10:52,714 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-04T15:10:52,714 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'ff648bb9ef17,41657,1733324995877' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-04T15:10:52,714 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-04T15:10:52,714 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-04T15:10:52,714 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-04T15:10:52,714 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-04T15:10:52,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T15:10:52,716 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,716 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,716 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:10:52,716 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:10:52,716 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:10:52,717 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T15:10:52,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:10:52,717 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:52,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:10:52,718 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:52,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,719 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:10:52,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:10:52,720 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:52,722 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T15:10:52,722 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T15:10:52,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:10:52,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-04T15:10:52,722 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T15:10:52,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-04T15:10:52,723 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T15:10:52,723 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:10:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-04T15:10:52,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-12-04T15:10:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325052735 (current time:1733325052735). 2024-12-04T15:10:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733325052734_backup_system_bulk VERSION not specified, setting to 2 2024-12-04T15:10:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22473bbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:52,737 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:52,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:52,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:52,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368d5912, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:52,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:52,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:52,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:52,738 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:52,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dc0050c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:52,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:52,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:52,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:52,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:52,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-12-04T15:10:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-12-04T15:10:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T15:10:52,744 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:52,745 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:52,747 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:52,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741914_1090 (size=76) 2024-12-04T15:10:52,770 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325038806 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325038806 2024-12-04T15:10:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T15:10:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T15:10:53,154 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:53,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ef965fc90d631230a5744cae57d35b0b}] 2024-12-04T15:10:53,155 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:53,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-12-04T15:10:53,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:53,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for ef965fc90d631230a5744cae57d35b0b: 2024-12-04T15:10:53,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. for snapshot_1733325052734_backup_system_bulk completed. 2024-12-04T15:10:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b.' region-info for snapshot=snapshot_1733325052734_backup_system_bulk 2024-12-04T15:10:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T15:10:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T15:10:53,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741915_1091 (size=61) 2024-12-04T15:10:53,321 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T15:10:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T15:10:53,471 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-04T15:10:53,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:10:53,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-04T15:10:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-04T15:10:53,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733325052734_backup_system_bulk on region ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:53,715 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:10:53,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-04T15:10:53,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure ef965fc90d631230a5744cae57d35b0b in 562 msec 2024-12-04T15:10:53,718 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:53,718 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:53,719 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:53,719 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733325052734_backup_system_bulk 2024-12-04T15:10:53,720 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325052734_backup_system_bulk 2024-12-04T15:10:53,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741916_1092 (size=702) 2024-12-04T15:10:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T15:10:54,131 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:10:54,136 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:10:54,136 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325052734_backup_system_bulk to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325052734_backup_system_bulk 2024-12-04T15:10:54,138 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:10:54,138 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-12-04T15:10:54,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1733325052734_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.3950 sec 2024-12-04T15:10:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T15:10:54,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-12-04T15:10:54,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } 2024-12-04T15:10:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325054893 (current time:1733325054893). 2024-12-04T15:10:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733325054892_ns4_test-17333250036803 VERSION not specified, setting to 2 2024-12-04T15:10:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d169df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:54,895 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:54,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:54,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:54,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6082873e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:54,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:54,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:54,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:54,896 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:54,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e1c7c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:54,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:54,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:54,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:54,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:54,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } 2024-12-04T15:10:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-12-04T15:10:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T15:10:54,902 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:54,903 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:54,905 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:54,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741917_1093 (size=86) 2024-12-04T15:10:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T15:10:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T15:10:55,312 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:55,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 031ac2bd7ebf036e19eeefd5230f16ab}] 2024-12-04T15:10:55,313 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:55,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-12-04T15:10:55,465 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:55,465 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 031ac2bd7ebf036e19eeefd5230f16ab: 2024-12-04T15:10:55,465 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. for snapshot_1733325054892_ns4_test-17333250036803 completed. 2024-12-04T15:10:55,465 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab.' region-info for snapshot=snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:10:55,465 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:55,465 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T15:10:55,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741918_1094 (size=52) 2024-12-04T15:10:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T15:10:55,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:10:55,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-04T15:10:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-04T15:10:55,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733325054892_ns4_test-17333250036803 on region 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:55,872 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:10:55,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-04T15:10:55,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 031ac2bd7ebf036e19eeefd5230f16ab in 561 msec 2024-12-04T15:10:55,875 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:55,876 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:55,877 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:55,877 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:10:55,877 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:10:55,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741919_1095 (size=443) 2024-12-04T15:10:55,887 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:10:55,891 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:10:55,892 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325054892_ns4_test-17333250036803 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:10:55,893 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:10:55,893 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-12-04T15:10:55,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1733325054892_ns4_test-17333250036803 table=ns4:test-17333250036803 type=FLUSH ttl=0 } in 992 msec 2024-12-04T15:10:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T15:10:56,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17333250036803 completed 2024-12-04T15:10:56,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } 2024-12-04T15:10:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325056043 (current time:1733325056043). 2024-12-04T15:10:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:56,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733325056043_default_test-1733325003680 VERSION not specified, setting to 2 2024-12-04T15:10:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562452e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:56,045 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:56,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:56,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:56,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e57d7b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:56,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:56,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:56,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:56,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:56,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a9a529b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:56,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:56,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:56,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:56,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:56,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } 2024-12-04T15:10:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-12-04T15:10:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T15:10:56,054 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:56,054 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:56,057 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:56,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741920_1096 (size=84) 2024-12-04T15:10:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T15:10:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T15:10:56,463 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:56,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44b146fc21bd97567dc283c76bd9e484}] 2024-12-04T15:10:56,464 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:56,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-12-04T15:10:56,616 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:56,616 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 44b146fc21bd97567dc283c76bd9e484: 2024-12-04T15:10:56,616 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. for snapshot_1733325056043_default_test-1733325003680 completed. 2024-12-04T15:10:56,616 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.' region-info for snapshot=snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:10:56,616 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:56,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/20099e47536a49fabaae486bca55e64a] hfiles 2024-12-04T15:10:56,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad for snapshot=snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:10:56,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/20099e47536a49fabaae486bca55e64a for snapshot=snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:10:56,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T15:10:56,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/40ffe374e1004bcc930d79fa22839b92] hfiles 2024-12-04T15:10:56,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c for snapshot=snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:10:56,617 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/40ffe374e1004bcc930d79fa22839b92 for snapshot=snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:10:56,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741921_1097 (size=224) 2024-12-04T15:10:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T15:10:57,023 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:10:57,023 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-04T15:10:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-04T15:10:57,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733325056043_default_test-1733325003680 on region 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:57,024 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:57,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-04T15:10:57,027 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:57,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 44b146fc21bd97567dc283c76bd9e484 in 562 msec 2024-12-04T15:10:57,027 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:57,029 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-04T15:10:57,029 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-04T15:10:57,029 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:57,029 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-12-04T15:10:57,029 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:57,030 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-12-04T15:10:57,030 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:10:57,031 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/mobdir/data/default/test-1733325003680/8f0eee486f65fbd2ba10e8355ebb380d/mob/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484] hfiles 2024-12-04T15:10:57,031 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/mobdir/data/default/test-1733325003680/8f0eee486f65fbd2ba10e8355ebb380d/mob/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:10:57,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741922_1098 (size=168) 2024-12-04T15:10:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T15:10:57,437 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:57,438 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:10:57,438 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:10:57,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741923_1099 (size=1347) 2024-12-04T15:10:57,848 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:10:57,855 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:10:57,856 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325056043_default_test-1733325003680 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:10:57,857 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:10:57,857 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-12-04T15:10:57,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1733325056043_default_test-1733325003680 table=test-1733325003680 type=FLUSH ttl=0 } in 1.8050 sec 2024-12-04T15:10:58,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T15:10:58,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1733325003680 completed 2024-12-04T15:10:58,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } 2024-12-04T15:10:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325058203 (current time:1733325058203). 2024-12-04T15:10:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:10:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733325058203_ns2_test-17333250036801 VERSION not specified, setting to 2 2024-12-04T15:10:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f5f734, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:10:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:10:58,205 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:10:58,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:10:58,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:10:58,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541f467b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:58,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:10:58,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:10:58,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:58,206 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:10:58,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b32efb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:10:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:10:58,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:10:58,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:10:58,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:10:58,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:10:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:10:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:10:58,210 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:10:58,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:10:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } 2024-12-04T15:10:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-12-04T15:10:58,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T15:10:58,213 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:10:58,214 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:10:58,216 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:10:58,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741924_1100 (size=86) 2024-12-04T15:10:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T15:10:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T15:10:58,625 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:10:58,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0896086d7d737ceb1077211e1712c9da}] 2024-12-04T15:10:58,626 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:58,686 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region e95ffbd37709d518765ec5c865d8f1ed, had cached 0 bytes from a total of 28925 2024-12-04T15:10:58,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-12-04T15:10:58,778 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:58,778 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 0896086d7d737ceb1077211e1712c9da 1/1 column families, dataSize=170 B heapSize=816 B 2024-12-04T15:10:58,782 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/.tmp/f/1b01658c740841febd21b2f55fb956b1 is 38, key is row-t20/f:q1/1733325033159/Put/seqid=0 2024-12-04T15:10:58,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741925_1101 (size=5134) 2024-12-04T15:10:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T15:10:59,187 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/.tmp/f/1b01658c740841febd21b2f55fb956b1 2024-12-04T15:10:59,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/.tmp/f/1b01658c740841febd21b2f55fb956b1 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/1b01658c740841febd21b2f55fb956b1 2024-12-04T15:10:59,199 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/1b01658c740841febd21b2f55fb956b1, entries=5, sequenceid=111, filesize=5.0 K 2024-12-04T15:10:59,200 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 0896086d7d737ceb1077211e1712c9da in 422ms, sequenceid=111, compaction requested=false 2024-12-04T15:10:59,201 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 0896086d7d737ceb1077211e1712c9da: 2024-12-04T15:10:59,201 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. for snapshot_1733325058203_ns2_test-17333250036801 completed. 2024-12-04T15:10:59,201 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da.' region-info for snapshot=snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:10:59,201 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:10:59,201 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/1b01658c740841febd21b2f55fb956b1] hfiles 2024-12-04T15:10:59,201 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314 for snapshot=snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:10:59,201 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/1b01658c740841febd21b2f55fb956b1 for snapshot=snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:10:59,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741926_1102 (size=130) 2024-12-04T15:10:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T15:10:59,402 INFO [master/ff648bb9ef17:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T15:10:59,402 INFO [master/ff648bb9ef17:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T15:10:59,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:10:59,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-04T15:10:59,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-04T15:10:59,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733325058203_ns2_test-17333250036801 on region 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:59,607 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:10:59,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-04T15:10:59,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 0896086d7d737ceb1077211e1712c9da in 983 msec 2024-12-04T15:10:59,610 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:10:59,611 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:10:59,611 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:10:59,611 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:10:59,612 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:10:59,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741927_1103 (size=496) 2024-12-04T15:11:00,022 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:11:00,028 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:11:00,029 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325058203_ns2_test-17333250036801 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:11:00,030 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:11:00,030 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-12-04T15:11:00,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1733325058203_ns2_test-17333250036801 table=ns2:test-17333250036801 type=FLUSH ttl=0 } in 1.8190 sec 2024-12-04T15:11:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T15:11:00,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17333250036801 completed 2024-12-04T15:11:00,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } 2024-12-04T15:11:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325060362 (current time:1733325060362). 2024-12-04T15:11:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:11:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1733325060362_ns3_test-17333250036802 VERSION not specified, setting to 2 2024-12-04T15:11:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799fa24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:11:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:11:00,364 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:11:00,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:11:00,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:11:00,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a68ee6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:00,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:11:00,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:11:00,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:00,366 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:11:00,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a36aac1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:11:00,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:11:00,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:11:00,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:11:00,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:11:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:11:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:00,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:11:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:11:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } 2024-12-04T15:11:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-12-04T15:11:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T15:11:00,372 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:11:00,373 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:11:00,375 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:11:00,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741928_1104 (size=86) 2024-12-04T15:11:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T15:11:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T15:11:00,781 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:11:00,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 505dfcba8179ccbcff79818fd950a242}] 2024-12-04T15:11:00,782 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:11:00,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-12-04T15:11:00,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:11:00,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 505dfcba8179ccbcff79818fd950a242: 2024-12-04T15:11:00,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. for snapshot_1733325060362_ns3_test-17333250036802 completed. 2024-12-04T15:11:00,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242.' region-info for snapshot=snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:00,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:11:00,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-12-04T15:11:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741929_1105 (size=52) 2024-12-04T15:11:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T15:11:01,341 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:11:01,341 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-04T15:11:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-04T15:11:01,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1733325060362_ns3_test-17333250036802 on region 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:11:01,342 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:11:01,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-04T15:11:01,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 505dfcba8179ccbcff79818fd950a242 in 562 msec 2024-12-04T15:11:01,344 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:11:01,345 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:11:01,345 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:11:01,345 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:01,346 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:01,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741930_1106 (size=443) 2024-12-04T15:11:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T15:11:01,756 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:11:01,761 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:11:01,762 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_1733325060362_ns3_test-17333250036802 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:01,763 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:11:01,763 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-12-04T15:11:01,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1733325060362_ns3_test-17333250036802 table=ns3:test-17333250036802 type=FLUSH ttl=0 } in 1.3930 sec 2024-12-04T15:11:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-04T15:11:02,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17333250036802 completed 2024-12-04T15:11:02,532 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1733325048418 2024-12-04T15:11:02,532 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-12-04T15:11:02,532 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-12-04T15:11:02,532 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733325048418_backup:system_bulk 2024-12-04T15:11:02,532 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733325052734_backup_system_bulk to hdfs://localhost:36003/backupUT/backup_1733325048418/backup/system_bulk/ 2024-12-04T15:11:02,532 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T15:11:02,565 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36003, inputRoot=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:11:02,565 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1922286205_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36003/backupUT/backup_1733325048418/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36003/backupUT/backup_1733325048418/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1733325052734_backup_system_bulk 2024-12-04T15:11:02,567 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T15:11:02,571 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325052734_backup_system_bulk to hdfs://localhost:36003/backupUT/backup_1733325048418/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1733325052734_backup_system_bulk 2024-12-04T15:11:02,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741931_1107 (size=76) 2024-12-04T15:11:02,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741932_1108 (size=702) 2024-12-04T15:11:02,982 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:02,983 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:02,983 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,978 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-6534330639138535136.jar 2024-12-04T15:11:03,978 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,979 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,979 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,979 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,979 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,980 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:03,981 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:11:03,981 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:11:03,981 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:11:03,981 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:11:03,982 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:11:03,982 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:11:03,982 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:11:03,982 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:11:03,982 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:11:03,983 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:11:03,983 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:11:03,983 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:03,984 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:03,984 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:03,984 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:03,984 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:03,984 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:03,985 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:04,026 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:11:04,054 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733325052734_backup_system_bulk' hfile list 2024-12-04T15:11:04,143 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local1638624727_0005_bce724f0-6fdd-4d70-9874-6c9e42ab6e38/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T15:11:04,143 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local1638624727_0005_bce724f0-6fdd-4d70-9874-6c9e42ab6e38/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T15:11:05,183 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T15:11:05,184 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T15:11:05,189 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733325052734_backup_system_bulk 2024-12-04T15:11:05,189 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733325052734_backup_system_bulk finished. 2024-12-04T15:11:05,189 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733325048418_ns4:test-17333250036803 2024-12-04T15:11:05,189 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733325054892_ns4_test-17333250036803 to hdfs://localhost:36003/backupUT/backup_1733325048418/ns4/test-17333250036803/ 2024-12-04T15:11:05,189 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T15:11:05,219 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36003, inputRoot=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:11:05,219 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1922286205_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36003/backupUT/backup_1733325048418/ns4/test-17333250036803, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36003/backupUT/backup_1733325048418/ns4/test-17333250036803/.hbase-snapshot/.tmp/snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:11:05,221 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T15:11:05,225 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325054892_ns4_test-17333250036803 to hdfs://localhost:36003/backupUT/backup_1733325048418/ns4/test-17333250036803/.hbase-snapshot/.tmp/snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:11:05,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741933_1109 (size=86) 2024-12-04T15:11:05,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741934_1110 (size=443) 2024-12-04T15:11:05,636 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:05,636 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:05,636 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,625 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-11695775865880644505.jar 2024-12-04T15:11:06,626 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,626 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,626 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,627 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,627 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,627 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,627 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,627 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,628 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:06,628 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:11:06,628 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:11:06,628 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:11:06,629 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:11:06,629 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:11:06,629 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:11:06,629 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:11:06,629 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:11:06,630 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:11:06,630 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:11:06,630 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:11:06,630 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:06,630 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:06,631 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:06,631 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:06,631 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:06,631 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:06,632 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:06,672 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:11:06,698 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733325054892_ns4_test-17333250036803' hfile list 2024-12-04T15:11:06,786 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local1709289466_0006_fbb477b2-3f19-47c5-a973-25e55f811e17/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T15:11:06,786 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local1709289466_0006_fbb477b2-3f19-47c5-a973-25e55f811e17/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T15:11:07,826 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T15:11:07,827 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T15:11:07,832 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:11:07,832 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733325054892_ns4_test-17333250036803 finished. 2024-12-04T15:11:07,832 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733325048418_test-1733325003680 2024-12-04T15:11:07,832 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733325056043_default_test-1733325003680 to hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/ 2024-12-04T15:11:07,832 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T15:11:07,864 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36003, inputRoot=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:11:07,864 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1922286205_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/.hbase-snapshot/.tmp/snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:11:07,866 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T15:11:07,872 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325056043_default_test-1733325003680 to hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/.hbase-snapshot/.tmp/snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:11:07,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741935_1111 (size=84) 2024-12-04T15:11:07,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741936_1112 (size=1347) 2024-12-04T15:11:08,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:08,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:08,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,280 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-14052510673332980083.jar 2024-12-04T15:11:09,281 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,281 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,281 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,281 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,282 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:09,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:11:09,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:11:09,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:11:09,283 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:11:09,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:11:09,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:11:09,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:11:09,284 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:11:09,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:11:09,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:11:09,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:11:09,285 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:09,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:09,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:09,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:09,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:09,286 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:09,287 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:09,330 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:11:09,358 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733325056043_default_test-1733325003680' hfile list 2024-12-04T15:11:09,360 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-12-04T15:11:09,451 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local30484138_0007_e7c7d372-6661-4377-81e1-ebfa7695f743/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T15:11:09,451 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local30484138_0007_e7c7d372-6661-4377-81e1-ebfa7695f743/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T15:11:09,522 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-04T15:11:09,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741937_1113 (size=13760) 2024-12-04T15:11:09,969 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/40ffe374e1004bcc930d79fa22839b92 output=hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/archive/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/40ffe374e1004bcc930d79fa22839b92 2024-12-04T15:11:09,969 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-12-04T15:11:09,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741938_1114 (size=10796) 2024-12-04T15:11:10,403 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/20099e47536a49fabaae486bca55e64a output=hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/archive/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/20099e47536a49fabaae486bca55e64a 2024-12-04T15:11:10,403 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-12-04T15:11:10,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741939_1115 (size=8413) 2024-12-04T15:11:10,845 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad output=hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/archive/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f/09d964a88b174a998307d9ec55b513ad 2024-12-04T15:11:10,845 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-12-04T15:11:10,870 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/mobdir/data/default/test-1733325003680/8f0eee486f65fbd2ba10e8355ebb380d/mob/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484 to hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/archive/data/default/test-1733325003680/8f0eee486f65fbd2ba10e8355ebb380d/mob/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484" truncated to max limit (512 characters) 2024-12-04T15:11:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741940_1116 (size=5591) 2024-12-04T15:11:11,278 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/mobdir/data/default/test-1733325003680/8f0eee486f65fbd2ba10e8355ebb380d/mob/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484 output=hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/archive/data/default/test-1733325003680/8f0eee486f65fbd2ba10e8355ebb380d/mob/d41d8cd98f00b204e9800998ecf8427e20241204625cd016e75b47618cfb721394cb1c6b_44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:11:11,279 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.012M/sec 2024-12-04T15:11:11,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741941_1117 (size=5484) 2024-12-04T15:11:11,320 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c output=hdfs://localhost:36003/backupUT/backup_1733325048418/default/test-1733325003680/archive/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/mob/d8fefa4a41ce454ab3752f28ab0e5e1c 2024-12-04T15:11:11,320 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.141M/sec 2024-12-04T15:11:11,493 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T15:11:11,494 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T15:11:11,502 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:11:11,502 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733325056043_default_test-1733325003680 finished. 2024-12-04T15:11:11,502 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733325048418_ns2:test-17333250036801 2024-12-04T15:11:11,502 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733325058203_ns2_test-17333250036801 to hdfs://localhost:36003/backupUT/backup_1733325048418/ns2/test-17333250036801/ 2024-12-04T15:11:11,502 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T15:11:11,532 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36003, inputRoot=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:11:11,532 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1922286205_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36003/backupUT/backup_1733325048418/ns2/test-17333250036801, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36003/backupUT/backup_1733325048418/ns2/test-17333250036801/.hbase-snapshot/.tmp/snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:11:11,534 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T15:11:11,538 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325058203_ns2_test-17333250036801 to hdfs://localhost:36003/backupUT/backup_1733325048418/ns2/test-17333250036801/.hbase-snapshot/.tmp/snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:11:11,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741942_1118 (size=86) 2024-12-04T15:11:11,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741943_1119 (size=496) 2024-12-04T15:11:11,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:11,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:11,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,944 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-7840261687673324099.jar 2024-12-04T15:11:12,945 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,945 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,945 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,946 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,946 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,946 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,946 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,946 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,947 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:12,947 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:11:12,947 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:11:12,947 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:11:12,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:11:12,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:11:12,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:11:12,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:11:12,948 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:11:12,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:11:12,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:11:12,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:11:12,949 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:12,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:12,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:12,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:12,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:12,950 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:12,951 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:12,991 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:11:13,017 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733325058203_ns2_test-17333250036801' hfile list 2024-12-04T15:11:13,019 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-12-04T15:11:13,104 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local1390197479_0008_c4461591-839c-41c2-84a2-cf5f0a4ab4a7/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T15:11:13,104 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local1390197479_0008_c4461591-839c-41c2-84a2-cf5f0a4ab4a7/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T15:11:13,175 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-12-04T15:11:13,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741944_1120 (size=8413) 2024-12-04T15:11:13,608 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314 output=hdfs://localhost:36003/backupUT/backup_1733325048418/ns2/test-17333250036801/archive/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/3e5cab4b92c342bd87890b494aa64314 2024-12-04T15:11:13,608 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.019M/sec 2024-12-04T15:11:13,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741945_1121 (size=5134) 2024-12-04T15:11:14,050 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/1b01658c740841febd21b2f55fb956b1 output=hdfs://localhost:36003/backupUT/backup_1733325048418/ns2/test-17333250036801/archive/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/f/1b01658c740841febd21b2f55fb956b1 2024-12-04T15:11:14,050 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-12-04T15:11:14,145 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T15:11:14,146 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T15:11:14,153 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:11:14,153 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733325058203_ns2_test-17333250036801 finished. 2024-12-04T15:11:14,153 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1733325048418_ns3:test-17333250036802 2024-12-04T15:11:14,153 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1733325060362_ns3_test-17333250036802 to hdfs://localhost:36003/backupUT/backup_1733325048418/ns3/test-17333250036802/ 2024-12-04T15:11:14,153 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-12-04T15:11:14,182 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:36003, inputRoot=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48 2024-12-04T15:11:14,182 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_-1922286205_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36003/backupUT/backup_1733325048418/ns3/test-17333250036802, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36003/backupUT/backup_1733325048418/ns3/test-17333250036802/.hbase-snapshot/.tmp/snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:14,184 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-12-04T15:11:14,187 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_1733325060362_ns3_test-17333250036802 to hdfs://localhost:36003/backupUT/backup_1733325048418/ns3/test-17333250036802/.hbase-snapshot/.tmp/snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741947_1123 (size=443) 2024-12-04T15:11:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741946_1122 (size=86) 2024-12-04T15:11:14,598 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:14,598 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:14,598 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,585 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-7117733431999079265.jar 2024-12-04T15:11:15,585 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,586 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,587 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:15,588 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:11:15,588 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:11:15,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:11:15,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:11:15,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:11:15,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:11:15,589 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:11:15,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:11:15,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:11:15,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:11:15,590 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:11:15,591 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:15,591 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:15,591 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:15,591 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:15,591 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-12-04T15:11:15,592 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:15,592 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:15,631 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:11:15,658 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1733325060362_ns3_test-17333250036802' hfile list 2024-12-04T15:11:15,747 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local22911324_0009_efbe6c38-486c-478a-b958-cc839a5f929e/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-12-04T15:11:15,747 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/mapreduce.cluster.local.dir/job_local22911324_0009_efbe6c38-486c-478a-b958-cc839a5f929e/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-12-04T15:11:16,787 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-12-04T15:11:16,788 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-12-04T15:11:16,793 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:16,793 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1733325060362_ns3_test-17333250036802 finished. 2024-12-04T15:11:16,793 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-12-04T15:11:16,793 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17333250036803 2024-12-04T15:11:16,793 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1733325003680 2024-12-04T15:11:16,793 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17333250036801 2024-12-04T15:11:16,793 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17333250036802 2024-12-04T15:11:16,802 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733325048418 2024-12-04T15:11:16,802 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-12-04T15:11:16,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741948_1124 (size=176) 2024-12-04T15:11:17,208 INFO [Time-limited test {}] impl.BackupManifest(504): Manifest file stored to hdfs://localhost:36003/backupUT/backup_1733325048418/.backup.manifest 2024-12-04T15:11:17,208 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1733325048418 finished: type=FULL,tablelist=backup:system_bulk;ns4:test-17333250036803;test-1733325003680;ns2:test-17333250036801;ns3:test-17333250036802,targetRootDir=hdfs://localhost:36003/backupUT,startts=1733325052613,completets=1733325076802,bytescopied=0 2024-12-04T15:11:17,208 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-12-04T15:11:17,208 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733325052734_backup_system_bulk 2024-12-04T15:11:17,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1733325052734_backup_system_bulk" type: DISABLED 2024-12-04T15:11:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733325052734_backup_system_bulk 2024-12-04T15:11:17,212 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733325052734_backup_system_bulk for backup backup_1733325048418 succeeded. 2024-12-04T15:11:17,212 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:11:17,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1733325054892_ns4_test-17333250036803" type: DISABLED 2024-12-04T15:11:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733325054892_ns4_test-17333250036803 2024-12-04T15:11:17,215 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733325054892_ns4_test-17333250036803 for backup backup_1733325048418 succeeded. 2024-12-04T15:11:17,215 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:11:17,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1733325056043_default_test-1733325003680" type: DISABLED 2024-12-04T15:11:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733325056043_default_test-1733325003680 2024-12-04T15:11:17,218 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733325056043_default_test-1733325003680 for backup backup_1733325048418 succeeded. 2024-12-04T15:11:17,218 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:11:17,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1733325058203_ns2_test-17333250036801" type: DISABLED 2024-12-04T15:11:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733325058203_ns2_test-17333250036801 2024-12-04T15:11:17,221 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733325058203_ns2_test-17333250036801 for backup backup_1733325048418 succeeded. 2024-12-04T15:11:17,221 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:17,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1733325060362_ns3_test-17333250036802" type: DISABLED 2024-12-04T15:11:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1733325060362_ns3_test-17333250036802 2024-12-04T15:11:17,224 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1733325060362_ns3_test-17333250036802 for backup backup_1733325048418 succeeded. 2024-12-04T15:11:17,225 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T15:11:17,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-12-04T15:11:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-04T15:11:17,231 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-04T15:11:17,233 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T15:11:17,238 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1733325048418 completed. 2024-12-04T15:11:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,296 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-12-04T15:11:22,300 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1733325082259 2024-12-04T15:11:22,302 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1733325048418} 2024-12-04T15:11:22,302 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-12-04T15:11:22,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:22,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:11:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325082387 (current time:1733325082387). 2024-12-04T15:11:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:11:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-12-04T15:11:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b81fdad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:11:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:11:22,389 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:11:22,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:11:22,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:11:22,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12315710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:22,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:11:22,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:11:22,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:22,390 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:11:22,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d27db3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:11:22,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:11:22,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:11:22,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:11:22,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:11:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:11:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:22,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:11:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:11:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:11:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-12-04T15:11:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T15:11:22,396 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:11:22,397 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:11:22,400 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:11:22,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741949_1125 (size=52) 2024-12-04T15:11:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T15:11:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T15:11:22,806 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-12-04T15:11:22,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed}] 2024-12-04T15:11:22,807 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:22,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-12-04T15:11:22,959 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:22,959 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing e95ffbd37709d518765ec5c865d8f1ed 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-12-04T15:11:22,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/9f6aee7887fa436e9b88887d3f3f3b6a is 158, key is trslm:hdfs://localhost:36003/backupUT\x00ns2:test-17333250036801/meta:log-roll-map/1733325076797/Put/seqid=0 2024-12-04T15:11:22,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741950_1126 (size=6646) 2024-12-04T15:11:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T15:11:23,321 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T15:11:23,369 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/9f6aee7887fa436e9b88887d3f3f3b6a 2024-12-04T15:11:23,378 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/da9cbdfa3d394c85836d56db51641928 is 826, key is session:backup_1733325048418/session:context/1733325077232/Put/seqid=0 2024-12-04T15:11:23,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741951_1127 (size=5928) 2024-12-04T15:11:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T15:11:23,782 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/da9cbdfa3d394c85836d56db51641928 2024-12-04T15:11:23,788 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/9f6aee7887fa436e9b88887d3f3f3b6a as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a 2024-12-04T15:11:23,792 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a, entries=12, sequenceid=42, filesize=6.5 K 2024-12-04T15:11:23,793 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/da9cbdfa3d394c85836d56db51641928 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928 2024-12-04T15:11:23,798 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928, entries=2, sequenceid=42, filesize=5.8 K 2024-12-04T15:11:23,799 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for e95ffbd37709d518765ec5c865d8f1ed in 839ms, sequenceid=42, compaction requested=true 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for e95ffbd37709d518765ec5c865d8f1ed: 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. for snapshot_backup_system completed. 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.' region-info for snapshot=snapshot_backup_system 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a] hfiles 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef for snapshot=snapshot_backup_system 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0 for snapshot=snapshot_backup_system 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a for snapshot=snapshot_backup_system 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928] hfiles 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 for snapshot=snapshot_backup_system 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620 for snapshot=snapshot_backup_system 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e for snapshot=snapshot_backup_system 2024-12-04T15:11:23,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928 for snapshot=snapshot_backup_system 2024-12-04T15:11:23,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741952_1128 (size=330) 2024-12-04T15:11:24,204 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:24,205 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-04T15:11:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-04T15:11:24,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:24,206 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:24,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-04T15:11:24,208 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-12-04T15:11:24,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure e95ffbd37709d518765ec5c865d8f1ed in 1.4000 sec 2024-12-04T15:11:24,209 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:11:24,209 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:11:24,209 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-12-04T15:11:24,210 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_backup_system 2024-12-04T15:11:24,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741953_1129 (size=967) 2024-12-04T15:11:24,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T15:11:24,620 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:11:24,628 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:11:24,628 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/snapshot_backup_system 2024-12-04T15:11:24,630 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:11:24,630 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-12-04T15:11:24,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2360 sec 2024-12-04T15:11:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-04T15:11:26,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T15:11:26,563 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1733325082259 started at 1733325086562. 2024-12-04T15:11:26,564 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1733325082259 has been started. 2024-12-04T15:11:26,569 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, ns2:test-17333250036801, ns3:test-17333250036802, ns4:test-17333250036803, test-1733325003680] 2024-12-04T15:11:26,572 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1733325038824for backupID backup_1733325082259 2024-12-04T15:11:26,572 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-12-04T15:11:26,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-12-04T15:11:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-12-04T15:11:26,573 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-12-04T15:11:26,573 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-12-04T15:11:26,573 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-12-04T15:11:26,573 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-12-04T15:11:26,574 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,574 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T15:11:26,580 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T15:11:26,581 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T15:11:26,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:11:26,581 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,581 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-12-04T15:11:26,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-12-04T15:11:26,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-12-04T15:11:26,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-12-04T15:11:26,582 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-12-04T15:11:26,582 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-12-04T15:11:26,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-12-04T15:11:26,583 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-12-04T15:11:26,583 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-12-04T15:11:26,583 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-12-04T15:11:26,583 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-12-04T15:11:26,583 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'ff648bb9ef17,41657,1733324995877' joining acquired barrier for procedure (rolllog) in zk 2024-12-04T15:11:26,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,585 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-12-04T15:11:26,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:11:26,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:11:26,585 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-12-04T15:11:26,585 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-12-04T15:11:26,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:11:26,586 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:11:26,586 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:11:26,586 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,587 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:11:26,587 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'ff648bb9ef17,41657,1733324995877' joining acquired barrier for procedure 'rolllog' on coordinator 2024-12-04T15:11:26,587 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@309185dd[Count = 0] remaining members to acquire global barrier 2024-12-04T15:11:26,587 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-12-04T15:11:26,587 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-12-04T15:11:26,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:11:26,590 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-12-04T15:11:26,590 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-12-04T15:11:26,590 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-12-04T15:11:26,590 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,590 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-12-04T15:11:26,590 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-12-04T15:11:26,590 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,590 INFO [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1733325052650 highest: 1733325052668 on ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,590 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.meta:.meta(num 1733325052631) roll requested 2024-12-04T15:11:26,604 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325086590.meta, exclude list is [], retry=0 2024-12-04T15:11:26,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:11:26,608 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325052631.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325086590.meta 2024-12-04T15:11:26,608 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:11:26,608 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325052631.meta is not closed yet, will try archiving it next time 2024-12-04T15:11:26,608 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0:(num 1733325052650) roll requested 2024-12-04T15:11:26,609 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325052631.meta not finished, retry = 0 2024-12-04T15:11:26,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741911_1087 (size=93) 2024-12-04T15:11:26,621 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325086608, exclude list is [], retry=0 2024-12-04T15:11:26,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:11:26,625 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325086608 2024-12-04T15:11:26,625 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:11:26,625 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 is not closed yet, will try archiving it next time 2024-12-04T15:11:26,625 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1:(num 1733325052668) roll requested 2024-12-04T15:11:26,625 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325019137 2024-12-04T15:11:26,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741912_1088 (size=653) 2024-12-04T15:11:26,627 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 not finished, retry = 0 2024-12-04T15:11:26,639 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325086625, exclude list is [], retry=0 2024-12-04T15:11:26,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38861,DS-b0ef5465-7cdd-4788-9674-8dcb6a35c037,DISK] 2024-12-04T15:11:26,642 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325052668 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325086625 2024-12-04T15:11:26,642 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38655:38655)] 2024-12-04T15:11:26,642 DEBUG [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325052668 is not closed yet, will try archiving it next time 2024-12-04T15:11:26,643 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325038824 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325038824 2024-12-04T15:11:26,644 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325052668 not finished, retry = 0 2024-12-04T15:11:26,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741913_1089 (size=4710) 2024-12-04T15:11:26,650 DEBUG [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-12-04T15:11:26,650 INFO [rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1733325086608 on ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-12-04T15:11:26,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:26,666 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-12-04T15:11:26,666 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'ff648bb9ef17,41657,1733324995877' in zk 2024-12-04T15:11:26,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,668 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-12-04T15:11:26,668 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T15:11:26,668 DEBUG [member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-12-04T15:11:26,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:11:26,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:11:26,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:11:26,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:11:26,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:11:26,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:11:26,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:11:26,670 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'ff648bb9ef17,41657,1733324995877': 2024-12-04T15:11:26,671 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'ff648bb9ef17,41657,1733324995877' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-12-04T15:11:26,671 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-12-04T15:11:26,671 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-12-04T15:11:26,671 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-12-04T15:11:26,671 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-12-04T15:11:26,671 INFO [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-12-04T15:11:26,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T15:11:26,673 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-12-04T15:11:26,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-12-04T15:11:26,673 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T15:11:26,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:11:26,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-12-04T15:11:26,674 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:11:26,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-12-04T15:11:26,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:11:26,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,675 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,675 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-12-04T15:11:26,676 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-12-04T15:11:26,676 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-12-04T15:11:26,678 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-12-04T15:11:26,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-12-04T15:11:26,678 DEBUG [(ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-12-04T15:11:26,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-12-04T15:11:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-12-04T15:11:26,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-12-04T15:11:26,679 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-12-04T15:11:26,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-12-04T15:11:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-12-04T15:11:26,684 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {ff648bb9ef17:41657=1733325038824} newestTimestamps: {ff648bb9ef17:41657=1733325052668} 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: ff648bb9ef17%2C41657%2C1733324995877.meta.1733324999110.meta 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: ff648bb9ef17%2C41657%2C1733324995877.meta.1733325038787.meta 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: ff648bb9ef17%2C41657%2C1733324995877.meta.1733325052631.meta 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: ff648bb9ef17%2C41657%2C1733324995877.meta.1733325086590.meta 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325086608 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325052668 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325086625 2024-12-04T15:11:26,686 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325019115.meta 2024-12-04T15:11:26,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741957_1133 (size=640) 2024-12-04T15:11:26,710 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325052631.meta to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325052631.meta 2024-12-04T15:11:26,727 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 2024-12-04T15:11:27,097 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:36003/backupUT/backup_1733325082259/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:11:27,097 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T15:11:27,099 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-12-04T15:11:27,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741958_1134 (size=45) 2024-12-04T15:11:27,504 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-12-04T15:11:27,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741959_1135 (size=390) 2024-12-04T15:11:27,915 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17333250036803 target: hdfs://localhost:36003/backupUT/backup_1733325082259/ns4/test-17333250036803 descriptor: 'ns4:test-17333250036803', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:11:27,915 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T15:11:27,918 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17333250036803 2024-12-04T15:11:27,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741960_1136 (size=50) 2024-12-04T15:11:28,323 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17333250036803 2024-12-04T15:11:28,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741961_1137 (size=950) 2024-12-04T15:11:28,734 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1733325003680 target: hdfs://localhost:36003/backupUT/backup_1733325082259/default/test-1733325003680 descriptor: 'test-1733325003680', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:11:28,734 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T15:11:28,737 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1733325003680 2024-12-04T15:11:28,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741962_1138 (size=53) 2024-12-04T15:11:29,142 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1733325003680 2024-12-04T15:11:29,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741963_1139 (size=364) 2024-12-04T15:11:29,553 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17333250036801 target: hdfs://localhost:36003/backupUT/backup_1733325082259/ns2/test-17333250036801 descriptor: 'ns2:test-17333250036801', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:11:29,553 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T15:11:29,555 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17333250036801 2024-12-04T15:11:29,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741964_1140 (size=50) 2024-12-04T15:11:29,961 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17333250036801 2024-12-04T15:11:29,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741965_1141 (size=390) 2024-12-04T15:11:30,372 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17333250036802 target: hdfs://localhost:36003/backupUT/backup_1733325082259/ns3/test-17333250036802 descriptor: 'ns3:test-17333250036802', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:11:30,372 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-12-04T15:11:30,374 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17333250036802 2024-12-04T15:11:30,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741966_1142 (size=50) 2024-12-04T15:11:30,779 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17333250036802 2024-12-04T15:11:30,788 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:36003/backupUT/.tmp/backup_1733325082259 from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650;hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1.1733325052668 2024-12-04T15:11:30,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cda9a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:30,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:11:30,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:11:30,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:11:30,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:11:30,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:11:30,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b219ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:30,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:11:30,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:11:30,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:30,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:11:30,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfd5cc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:30,792 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:11:30,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is ff648bb9ef17,42357,1733324994229 2024-12-04T15:11:30,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2852b3f4 2024-12-04T15:11:30,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:11:30,794 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T15:11:30,796 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-12-04T15:11:30,797 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-12-04T15:11:30,797 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-12-04T15:11:30,797 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:11:30,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:11:30,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:11:30,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:11:30,803 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-12-04T15:11:30,803 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17333250036801 2024-12-04T15:11:30,804 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17333250036801,, stopping at row=ns2:test-17333250036801 ,, for max=2147483647 with caching=100 2024-12-04T15:11:30,805 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17333250036801: ns2:test-17333250036801; 2024-12-04T15:11:30,805 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17333250036802 2024-12-04T15:11:30,805 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17333250036802,, stopping at row=ns3:test-17333250036802 ,, for max=2147483647 with caching=100 2024-12-04T15:11:30,807 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17333250036802: ns3:test-17333250036802; 2024-12-04T15:11:30,807 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17333250036803 2024-12-04T15:11:30,807 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17333250036803,, stopping at row=ns4:test-17333250036803 ,, for max=2147483647 with caching=100 2024-12-04T15:11:30,808 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17333250036803: ns4:test-17333250036803; 2024-12-04T15:11:30,808 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1733325003680 2024-12-04T15:11:30,808 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1733325003680,, stopping at row=test-1733325003680 ,, for max=2147483647 with caching=100 2024-12-04T15:11:30,810 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1733325003680: test-1733325003680; 2024-12-04T15:11:30,810 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-12-04T15:11:30,810 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/hbase-staging/partitions_8c96a71c-65b6-4bd4-b221-edf4af670462 2024-12-04T15:11:30,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741967_1143 (size=324) 2024-12-04T15:11:30,817 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:30,817 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:30,818 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,803 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-15629199941974653025.jar 2024-12-04T15:11:31,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,804 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,805 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,805 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,805 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,806 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,806 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,806 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,807 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,807 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:11:31,808 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-12-04T15:11:31,808 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-12-04T15:11:31,808 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-12-04T15:11:31,809 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-12-04T15:11:31,809 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-12-04T15:11:31,809 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-12-04T15:11:31,809 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-12-04T15:11:31,809 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-12-04T15:11:31,810 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-12-04T15:11:31,810 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-12-04T15:11:31,810 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,810 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,811 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,811 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,811 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,811 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-12-04T15:11:31,811 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-12-04T15:11:31,812 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,ns2:test-17333250036801,ns3:test-17333250036802,ns4:test-17333250036803,default:test-1733325003680 2024-12-04T15:11:31,812 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:11:31,812 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:188) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:11:31,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:31,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:31,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:11:31,812 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-12-04T15:11:32,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop-11522283209900379439.jar 2024-12-04T15:11:32,809 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-12-04T15:11:32,835 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 for WAL files 2024-12-04T15:11:32,836 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1733325082259File hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 does not exist. java.io.FileNotFoundException: File hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:188) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:11:32,837 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1733325082259,startts=1733325086562,failedts=1733325092837,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325052650 does not exist. 2024-12-04T15:11:32,837 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-12-04T15:11:32,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable backup:system 2024-12-04T15:11:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-12-04T15:11:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T15:11:32,854 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325092853"}]},"ts":"1733325092853"} 2024-12-04T15:11:32,855 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-12-04T15:11:32,855 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-12-04T15:11:32,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-12-04T15:11:32,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, UNASSIGN}] 2024-12-04T15:11:32,863 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, UNASSIGN 2024-12-04T15:11:32,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e95ffbd37709d518765ec5c865d8f1ed, regionState=CLOSING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:32,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, UNASSIGN because future has completed 2024-12-04T15:11:32,866 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T15:11:32,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:11:32,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T15:11:33,018 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:33,019 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T15:11:33,019 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing e95ffbd37709d518765ec5c865d8f1ed, disabling compactions & flushes 2024-12-04T15:11:33,019 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:33,019 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:33,019 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. after waiting 0 ms 2024-12-04T15:11:33,019 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:33,019 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing e95ffbd37709d518765ec5c865d8f1ed 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-12-04T15:11:33,023 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/afcfd5f306ba4055bcece193915db028 is 103, key is rslogts:hdfs://localhost:36003/backupUT\x00ff648bb9ef17:41657/meta:rs-log-ts/1733325086665/Put/seqid=0 2024-12-04T15:11:33,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741968_1144 (size=5338) 2024-12-04T15:11:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T15:11:33,264 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 44b146fc21bd97567dc283c76bd9e484, had cached 0 bytes from a total of 38453 2024-12-04T15:11:33,428 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/afcfd5f306ba4055bcece193915db028 2024-12-04T15:11:33,436 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/96fcc9d9fe9042dfab8986272dde3ddc is 303, key is session:backup_1733325082259/session:context/1733325086563/Put/seqid=0 2024-12-04T15:11:33,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741969_1145 (size=5391) 2024-12-04T15:11:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T15:11:33,840 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/96fcc9d9fe9042dfab8986272dde3ddc 2024-12-04T15:11:33,846 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/afcfd5f306ba4055bcece193915db028 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/afcfd5f306ba4055bcece193915db028 2024-12-04T15:11:33,851 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/afcfd5f306ba4055bcece193915db028, entries=1, sequenceid=47, filesize=5.2 K 2024-12-04T15:11:33,852 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/96fcc9d9fe9042dfab8986272dde3ddc as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/96fcc9d9fe9042dfab8986272dde3ddc 2024-12-04T15:11:33,856 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/96fcc9d9fe9042dfab8986272dde3ddc, entries=1, sequenceid=47, filesize=5.3 K 2024-12-04T15:11:33,857 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for e95ffbd37709d518765ec5c865d8f1ed in 838ms, sequenceid=47, compaction requested=true 2024-12-04T15:11:33,861 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-12-04T15:11:33,862 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:11:33,862 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:33,863 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for e95ffbd37709d518765ec5c865d8f1ed: Waiting for close lock at 1733325093019Running coprocessor pre-close hooks at 1733325093019Disabling compacts and flushes for region at 1733325093019Disabling writes for close at 1733325093019Obtaining lock to block concurrent updates at 1733325093019Preparing flush snapshotting stores in e95ffbd37709d518765ec5c865d8f1ed at 1733325093019Finished memstore snapshotting backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1733325093019Flushing stores of backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. at 1733325093020 (+1 ms)Flushing e95ffbd37709d518765ec5c865d8f1ed/meta: creating writer at 1733325093020Flushing e95ffbd37709d518765ec5c865d8f1ed/meta: appending metadata at 1733325093023 (+3 ms)Flushing e95ffbd37709d518765ec5c865d8f1ed/meta: closing flushed file at 1733325093023Flushing e95ffbd37709d518765ec5c865d8f1ed/session: creating writer at 1733325093433 (+410 ms)Flushing e95ffbd37709d518765ec5c865d8f1ed/session: appending metadata at 1733325093435 (+2 ms)Flushing e95ffbd37709d518765ec5c865d8f1ed/session: closing flushed file at 1733325093436 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49672171: reopening flushed file at 1733325093845 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51831b97: reopening flushed file at 1733325093851 (+6 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for e95ffbd37709d518765ec5c865d8f1ed in 838ms, sequenceid=47, compaction requested=true at 1733325093857 (+6 ms)Writing region close event to WAL at 1733325093859 (+2 ms)Running coprocessor post-close hooks at 1733325093862 (+3 ms)Closed at 1733325093862 2024-12-04T15:11:33,864 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:33,865 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e95ffbd37709d518765ec5c865d8f1ed, regionState=CLOSED 2024-12-04T15:11:33,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:11:33,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-04T15:11:33,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877 in 1.0020 sec 2024-12-04T15:11:33,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-04T15:11:33,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, UNASSIGN in 1.0070 sec 2024-12-04T15:11:33,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-04T15:11:33,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0150 sec 2024-12-04T15:11:33,874 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325093874"}]},"ts":"1733325093874"} 2024-12-04T15:11:33,876 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-12-04T15:11:33,876 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-12-04T15:11:33,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0300 sec 2024-12-04T15:11:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-04T15:11:33,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-12-04T15:11:33,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1733325093999 2024-12-04T15:11:34,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:11:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1733325094000 (current time:1733325094000). 2024-12-04T15:11:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-12-04T15:11:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1733325093999 VERSION not specified, setting to 2 2024-12-04T15:11:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2972d92d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ClusterIdFetcher(90): Going to request ff648bb9ef17,42357,-1 for getting cluster id 2024-12-04T15:11:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:11:34,002 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ce837f02-95e7-4231-a6b7-9a4367546947' 2024-12-04T15:11:34,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:11:34,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ce837f02-95e7-4231-a6b7-9a4367546947" 2024-12-04T15:11:34,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f63547b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:34,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ff648bb9ef17,42357,-1] 2024-12-04T15:11:34,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:11:34,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:34,003 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:11:34,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3d58c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-12-04T15:11:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:11:34,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ff648bb9ef17,41657,1733324995877, seqNum=-1] 2024-12-04T15:11:34,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:11:34,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:11:34,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357. 2024-12-04T15:11:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:11:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:34,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:11:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-12-04T15:11:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:11:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-12-04T15:11:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T15:11:34,009 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-12-04T15:11:34,010 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-12-04T15:11:34,012 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-12-04T15:11:34,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741970_1146 (size=81) 2024-12-04T15:11:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T15:11:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T15:11:34,418 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-12-04T15:11:34,419 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-12-04T15:11:34,419 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-12-04T15:11:34,419 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:11:34,421 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/afcfd5f306ba4055bcece193915db028, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0] hfiles 2024-12-04T15:11:34,421 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef 2024-12-04T15:11:34,421 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a 2024-12-04T15:11:34,421 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/afcfd5f306ba4055bcece193915db028 2024-12-04T15:11:34,421 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0 2024-12-04T15:11:34,421 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:11:34,423 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/96fcc9d9fe9042dfab8986272dde3ddc, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e] hfiles 2024-12-04T15:11:34,423 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620 2024-12-04T15:11:34,423 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 2024-12-04T15:11:34,423 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/96fcc9d9fe9042dfab8986272dde3ddc 2024-12-04T15:11:34,423 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928 2024-12-04T15:11:34,423 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e 2024-12-04T15:11:34,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741971_1147 (size=409) 2024-12-04T15:11:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T15:11:34,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-12-04T15:11:34,829 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => e95ffbd37709d518765ec5c865d8f1ed, NAME => 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-12-04T15:11:34,830 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-12-04T15:11:34,831 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-12-04T15:11:34,831 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1733325093999 2024-12-04T15:11:34,831 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1733325093999 2024-12-04T15:11:34,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741972_1148 (size=1046) 2024-12-04T15:11:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T15:11:35,241 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-12-04T15:11:35,250 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-12-04T15:11:35,250 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1733325093999 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1733325093999 2024-12-04T15:11:35,252 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-12-04T15:11:35,252 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-12-04T15:11:35,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1733325093999 table=backup:system type=FLUSH ttl=0 } in 1.2450 sec 2024-12-04T15:11:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-04T15:11:36,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-12-04T15:11:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1733325082387 type: FLUSH version: 2 ttl: 0 ) 2024-12-04T15:11:36,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-12-04T15:11:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T15:11:36,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741973_1149 (size=635) 2024-12-04T15:11:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T15:11:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T15:11:36,584 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-12-04T15:11:36,584 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-12-04T15:11:36,585 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-12-04T15:11:36,590 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1733325082387 type: FLUSH version: 2 ttl: 0 2024-12-04T15:11:36,590 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system 2024-12-04T15:11:36,593 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-12-04T15:11:36,593 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(229): region to restore: e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:36,597 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:11:36,600 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/afcfd5f306ba4055bcece193915db028 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/afcfd5f306ba4055bcece193915db028 2024-12-04T15:11:36,600 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:11:36,603 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/96fcc9d9fe9042dfab8986272dde3ddc to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/96fcc9d9fe9042dfab8986272dde3ddc 2024-12-04T15:11:36,604 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1733325082387 type: FLUSH version: 2 ttl: 0 2024-12-04T15:11:36,605 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733325096605"}]},"ts":"9223372036854775807"} 2024-12-04T15:11:36,608 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-04T15:11:36,608 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e95ffbd37709d518765ec5c865d8f1ed, NAME => 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T15:11:36,608 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1733325096606"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733325096606"}]},"ts":"1733325096606"} 2024-12-04T15:11:36,610 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:11:36,610 INFO [PEWorker-5 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-12-04T15:11:36,610 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => e95ffbd37709d518765ec5c865d8f1ed, NAME => 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.', STARTKEY => '', ENDKEY => ''}] 2024-12-04T15:11:36,611 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-12-04T15:11:36,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1733325082387 type: FLUSH version: 2 ttl: 0 ) in 446 msec 2024-12-04T15:11:36,657 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 0896086d7d737ceb1077211e1712c9da, had cached 0 bytes from a total of 13547 2024-12-04T15:11:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T15:11:36,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1733325093999 2024-12-04T15:11:36,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "hbase-failsafe-snapshot_backup_system-1733325093999" type: DISABLED 2024-12-04T15:11:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1733325093999 2024-12-04T15:11:36,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.2 enable backup:system 2024-12-04T15:11:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-12-04T15:11:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T15:11:36,818 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-12-04T15:11:36,819 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325096818"}]},"ts":"1733325096818"} 2024-12-04T15:11:36,820 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-12-04T15:11:36,821 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-12-04T15:11:36,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN}] 2024-12-04T15:11:36,822 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN 2024-12-04T15:11:36,823 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-12-04T15:11:36,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T15:11:36,973 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=e95ffbd37709d518765ec5c865d8f1ed, regionState=OPENING, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:36,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN because future has completed 2024-12-04T15:11:36,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877}] 2024-12-04T15:11:37,131 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,131 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => e95ffbd37709d518765ec5c865d8f1ed, NAME => 'backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:11:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T15:11:37,132 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-12-04T15:11:37,132 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,132 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:11:37,133 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,133 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,134 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,135 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e95ffbd37709d518765ec5c865d8f1ed columnFamilyName meta 2024-12-04T15:11:37,135 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:11:37,141 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef 2024-12-04T15:11:37,146 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a 2024-12-04T15:11:37,150 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0 2024-12-04T15:11:37,150 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.HStore(327): Store=e95ffbd37709d518765ec5c865d8f1ed/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:11:37,150 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,151 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e95ffbd37709d518765ec5c865d8f1ed columnFamilyName session 2024-12-04T15:11:37,151 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:11:37,157 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620 2024-12-04T15:11:37,161 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 2024-12-04T15:11:37,164 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928 2024-12-04T15:11:37,168 DEBUG [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e 2024-12-04T15:11:37,168 INFO [StoreOpener-e95ffbd37709d518765ec5c865d8f1ed-1 {}] regionserver.HStore(327): Store=e95ffbd37709d518765ec5c865d8f1ed/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:11:37,168 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,169 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,170 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,170 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,170 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,171 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T15:11:37,172 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,173 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened e95ffbd37709d518765ec5c865d8f1ed; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11265337760, jitterRate=0.04916633665561676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T15:11:37,173 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,173 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for e95ffbd37709d518765ec5c865d8f1ed: Running coprocessor pre-open hook at 1733325097133Writing region info on filesystem at 1733325097133Initializing all the Stores at 1733325097134 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325097134Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325097134Cleaning up temporary data from old regions at 1733325097170 (+36 ms)Running coprocessor post-open hooks at 1733325097173 (+3 ms)Region opened successfully at 1733325097173 2024-12-04T15:11:37,174 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., pid=70, masterSystemTime=1733325097127 2024-12-04T15:11:37,176 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store e95ffbd37709d518765ec5c865d8f1ed:meta, priority=-2147483648, current under compaction store size is 1 2024-12-04T15:11:37,177 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:11:37,177 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store e95ffbd37709d518765ec5c865d8f1ed:session, priority=-2147483648, current under compaction store size is 2 2024-12-04T15:11:37,177 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-04T15:11:37,178 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-04T15:11:37,178 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T15:11:37,179 DEBUG [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,179 INFO [RS_OPEN_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,180 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-04T15:11:37,180 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-04T15:11:37,180 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=e95ffbd37709d518765ec5c865d8f1ed, regionState=OPEN, openSeqNum=51, regionLocation=ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:37,181 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.HStore(1541): e95ffbd37709d518765ec5c865d8f1ed/session is initiating minor compaction (all files) 2024-12-04T15:11:37,181 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.HStore(1541): e95ffbd37709d518765ec5c865d8f1ed/meta is initiating minor compaction (all files) 2024-12-04T15:11:37,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877 because future has completed 2024-12-04T15:11:37,182 INFO [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e95ffbd37709d518765ec5c865d8f1ed/meta in backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,182 INFO [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e95ffbd37709d518765ec5c865d8f1ed/session in backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,182 INFO [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928] into tmpdir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp, totalSize=22.7 K 2024-12-04T15:11:37,182 INFO [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a] into tmpdir=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp, totalSize=17.9 K 2024-12-04T15:11:37,183 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7878fc449f4a44fe851a48d3494268ef, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733325019220 2024-12-04T15:11:37,183 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d6d5ab9ccc944e9a56bb59a9e20bb85, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733325016886 2024-12-04T15:11:37,183 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3329fbe4f474becb485a64cbd0f94b0, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733325038863 2024-12-04T15:11:37,184 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a6b3bf73454805ba3274fdb0479620, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733325032758 2024-12-04T15:11:37,184 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f6aee7887fa436e9b88887d3f3f3b6a, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733325052706 2024-12-04T15:11:37,184 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] compactions.Compactor(225): Compacting f3a833403bfc4f00b96e0f5a1f10406e, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733325047337 2024-12-04T15:11:37,185 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] compactions.Compactor(225): Compacting da9cbdfa3d394c85836d56db51641928, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1733325077232 2024-12-04T15:11:37,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-12-04T15:11:37,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure e95ffbd37709d518765ec5c865d8f1ed, server=ff648bb9ef17,41657,1733324995877 in 207 msec 2024-12-04T15:11:37,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-04T15:11:37,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=e95ffbd37709d518765ec5c865d8f1ed, ASSIGN in 364 msec 2024-12-04T15:11:37,188 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325097187"}]},"ts":"1733325097187"} 2024-12-04T15:11:37,189 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-12-04T15:11:37,189 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-12-04T15:11:37,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 378 msec 2024-12-04T15:11:37,211 INFO [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e95ffbd37709d518765ec5c865d8f1ed#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:11:37,211 INFO [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e95ffbd37709d518765ec5c865d8f1ed#meta#compaction#16 average throughput is 1.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T15:11:37,212 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/957a17f8b7714df7b5d93d07833dea1f is 158, key is trslm:hdfs://localhost:36003/backupUT\x00ns2:test-17333250036801/meta:log-roll-map/1733325076797/Put/seqid=0 2024-12-04T15:11:37,212 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/bfb20bc44f954b128e57fd363f424486 is 1245, key is session:backup_1733325048326/session:context/1733325048413/Put/seqid=0 2024-12-04T15:11:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741975_1151 (size=6749) 2024-12-04T15:11:37,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741974_1150 (size=8073) 2024-12-04T15:11:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T15:11:37,443 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-12-04T15:11:37,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-12-04T15:11:37,443 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-12-04T15:11:37,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-12-04T15:11:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42357 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-12-04T15:11:37,450 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-12-04T15:11:37,450 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-12-04T15:11:37,452 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-12-04T15:11:37,457 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1733325082259 failed. 2024-12-04T15:11:37,457 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:11:37,458 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:11:37,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:37,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:37,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:11:37,477 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=627 (was 611) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-987479468_20 at /127.0.0.1:34126 [Receiving block BP-2007428057-172.17.0.2-1733324991063:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2007428057-172.17.0.2-1733324991063:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2007428057-172.17.0.2-1733324991063:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: member: 'ff648bb9ef17,41657,1733324995877' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_207481731_20 at /127.0.0.1:34058 [Waiting for operation #71] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/957a17f8b7714df7b5d93d07833dea1f java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-987479468_20 at /127.0.0.1:34116 [Receiving block BP-2007428057-172.17.0.2-1733324991063:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (ff648bb9ef17,42357,1733324994229)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2007428057-172.17.0.2-1733324991063:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/bfb20bc44f954b128e57fd363f424486 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-987479468_20 at /127.0.0.1:34138 [Receiving block BP-2007428057-172.17.0.2-1733324991063:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(ff648bb9ef17,41657,1733324995877)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ff648bb9ef17:41657-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/ff648bb9ef17:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=641 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=101 (was 179), ProcessCount=12 (was 12), AvailableMemoryMB=10268 (was 11144) 2024-12-04T15:11:37,479 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=627 is superior to 500 2024-12-04T15:11:37,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T15:11:37,485 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:11:37,485 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:11:37,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:37,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:37,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:11:37,486 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T15:11:37,486 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1774179625, stopped=false 2024-12-04T15:11:37,486 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ff648bb9ef17,42357,1733324994229 2024-12-04T15:11:37,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-12-04T15:11:37,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-12-04T15:11:37,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:11:37,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:11:37,488 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T15:11:37,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:11:37,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:11:37,489 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-04T15:11:37,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:37,489 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-12-04T15:11:37,489 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ff648bb9ef17,41657,1733324995877' ***** 2024-12-04T15:11:37,489 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T15:11:37,490 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.HBaseServerBase(468): Stop info server 2024-12-04T15:11:37,494 INFO [RS:0;ff648bb9ef17:41657 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@530459fd{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-12-04T15:11:37,496 INFO [RS:0;ff648bb9ef17:41657 {}] server.AbstractConnector(383): Stopped ServerConnector@21b68432{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-04T15:11:37,496 INFO [RS:0;ff648bb9ef17:41657 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:11:37,496 INFO [RS:0;ff648bb9ef17:41657 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6e82f00d{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-12-04T15:11:37,496 INFO [RS:0;ff648bb9ef17:41657 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3f822913{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,STOPPED} 2024-12-04T15:11:37,498 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T15:11:37,498 INFO [RS:0;ff648bb9ef17:41657 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T15:11:37,498 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T15:11:37,498 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-12-04T15:11:37,498 INFO [RS:0;ff648bb9ef17:41657 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T15:11:37,498 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(3091): Received CLOSE for 505dfcba8179ccbcff79818fd950a242 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(3091): Received CLOSE for ef965fc90d631230a5744cae57d35b0b 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(3091): Received CLOSE for 0896086d7d737ceb1077211e1712c9da 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(3091): Received CLOSE for e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(3091): Received CLOSE for 031ac2bd7ebf036e19eeefd5230f16ab 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(3091): Received CLOSE for 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(959): stopping server ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ff648bb9ef17:41657. 2024-12-04T15:11:37,499 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 505dfcba8179ccbcff79818fd950a242, disabling compactions & flushes 2024-12-04T15:11:37,499 DEBUG [RS:0;ff648bb9ef17:41657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:11:37,499 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:11:37,499 DEBUG [RS:0;ff648bb9ef17:41657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:37,499 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:11:37,499 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. after waiting 0 ms 2024-12-04T15:11:37,499 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:11:37,499 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T15:11:37,503 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns3/test-17333250036802/505dfcba8179ccbcff79818fd950a242/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T15:11:37,503 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:11:37,503 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:11:37,503 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 505dfcba8179ccbcff79818fd950a242: Waiting for close lock at 1733325097499Running coprocessor pre-close hooks at 1733325097499Disabling compacts and flushes for region at 1733325097499Disabling writes for close at 1733325097499Writing region close event to WAL at 1733325097499Running coprocessor post-close hooks at 1733325097503 (+4 ms)Closed at 1733325097503 2024-12-04T15:11:37,503 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17333250036802,,1733325007790.505dfcba8179ccbcff79818fd950a242. 2024-12-04T15:11:37,503 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef965fc90d631230a5744cae57d35b0b, disabling compactions & flushes 2024-12-04T15:11:37,504 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:11:37,504 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:11:37,504 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. after waiting 0 ms 2024-12-04T15:11:37,504 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:11:37,507 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system_bulk/ef965fc90d631230a5744cae57d35b0b/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T15:11:37,507 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:11:37,507 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:11:37,507 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef965fc90d631230a5744cae57d35b0b: Waiting for close lock at 1733325097503Running coprocessor pre-close hooks at 1733325097503Disabling compacts and flushes for region at 1733325097503Disabling writes for close at 1733325097504 (+1 ms)Writing region close event to WAL at 1733325097504Running coprocessor post-close hooks at 1733325097507 (+3 ms)Closed at 1733325097507 2024-12-04T15:11:37,507 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1733325014703.ef965fc90d631230a5744cae57d35b0b. 2024-12-04T15:11:37,507 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0896086d7d737ceb1077211e1712c9da, disabling compactions & flushes 2024-12-04T15:11:37,507 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:11:37,508 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:11:37,508 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. after waiting 0 ms 2024-12-04T15:11:37,508 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:11:37,509 INFO [regionserver/ff648bb9ef17:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T15:11:37,511 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns2/test-17333250036801/0896086d7d737ceb1077211e1712c9da/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-12-04T15:11:37,511 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:11:37,511 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:11:37,511 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0896086d7d737ceb1077211e1712c9da: Waiting for close lock at 1733325097507Running coprocessor pre-close hooks at 1733325097507Disabling compacts and flushes for region at 1733325097507Disabling writes for close at 1733325097508 (+1 ms)Writing region close event to WAL at 1733325097508Running coprocessor post-close hooks at 1733325097511 (+3 ms)Closed at 1733325097511 2024-12-04T15:11:37,511 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da. 2024-12-04T15:11:37,512 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e95ffbd37709d518765ec5c865d8f1ed, disabling compactions & flushes 2024-12-04T15:11:37,512 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,625 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/bfb20bc44f954b128e57fd363f424486 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/bfb20bc44f954b128e57fd363f424486 2024-12-04T15:11:37,625 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/meta/957a17f8b7714df7b5d93d07833dea1f as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/957a17f8b7714df7b5d93d07833dea1f 2024-12-04T15:11:37,636 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-12-04T15:11:37,636 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-12-04T15:11:37,639 INFO [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e95ffbd37709d518765ec5c865d8f1ed/meta of e95ffbd37709d518765ec5c865d8f1ed into 957a17f8b7714df7b5d93d07833dea1f(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:11:37,639 INFO [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e95ffbd37709d518765ec5c865d8f1ed/session of e95ffbd37709d518765ec5c865d8f1ed into bfb20bc44f954b128e57fd363f424486(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T15:11:37,639 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e95ffbd37709d518765ec5c865d8f1ed: 2024-12-04T15:11:37,639 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e95ffbd37709d518765ec5c865d8f1ed: 2024-12-04T15:11:37,639 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,639 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,639 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. after waiting 0 ms 2024-12-04T15:11:37,639 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:37,639 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e95ffbd37709d518765ec5c865d8f1ed 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-12-04T15:11:37,641 INFO [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., storeName=e95ffbd37709d518765ec5c865d8f1ed/meta, priority=13, startTime=1733325097175; duration=0sec 2024-12-04T15:11:37,641 INFO [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., storeName=e95ffbd37709d518765ec5c865d8f1ed/session, priority=12, startTime=1733325097177; duration=0sec 2024-12-04T15:11:37,641 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:11:37,641 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-04T15:11:37,642 DEBUG [RS:0;ff648bb9ef17:41657-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e95ffbd37709d518765ec5c865d8f1ed:meta 2024-12-04T15:11:37,642 DEBUG [RS:0;ff648bb9ef17:41657-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e95ffbd37709d518765ec5c865d8f1ed:session 2024-12-04T15:11:37,642 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T15:11:37,642 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T15:11:37,642 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T15:11:37,642 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-04T15:11:37,642 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T15:11:37,642 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1325): Online Regions={e95ffbd37709d518765ec5c865d8f1ed=backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., 1588230740=hbase:meta,,1.1588230740, 031ac2bd7ebf036e19eeefd5230f16ab=ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab., 44b146fc21bd97567dc283c76bd9e484=test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484.} 2024-12-04T15:11:37,642 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T15:11:37,643 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T15:11:37,643 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T15:11:37,643 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T15:11:37,643 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1351): Waiting on 031ac2bd7ebf036e19eeefd5230f16ab, 1588230740, 44b146fc21bd97567dc283c76bd9e484, e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:37,643 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-12-04T15:11:37,644 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/acccb9d030aa44a3a9637c2a84e368fd is 554, key is session:backup_1733325082259/session:context/1733325097451/Put/seqid=0 2024-12-04T15:11:37,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741976_1152 (size=5655) 2024-12-04T15:11:37,666 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/info/8372257e88ed497dadaf5c1f91341425 is 159, key is ns2:test-17333250036801,,1733325005502.0896086d7d737ceb1077211e1712c9da./info:regioninfo/1733325006677/Put/seqid=0 2024-12-04T15:11:37,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741977_1153 (size=14126) 2024-12-04T15:11:37,843 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1351): Waiting on 031ac2bd7ebf036e19eeefd5230f16ab, 1588230740, 44b146fc21bd97567dc283c76bd9e484, e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:38,043 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1351): Waiting on 031ac2bd7ebf036e19eeefd5230f16ab, 1588230740, 44b146fc21bd97567dc283c76bd9e484, e95ffbd37709d518765ec5c865d8f1ed 2024-12-04T15:11:38,049 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/acccb9d030aa44a3a9637c2a84e368fd 2024-12-04T15:11:38,055 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/.tmp/session/acccb9d030aa44a3a9637c2a84e368fd as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/acccb9d030aa44a3a9637c2a84e368fd 2024-12-04T15:11:38,060 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/acccb9d030aa44a3a9637c2a84e368fd, entries=2, sequenceid=57, filesize=5.5 K 2024-12-04T15:11:38,061 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for e95ffbd37709d518765ec5c865d8f1ed in 422ms, sequenceid=57, compaction requested=false 2024-12-04T15:11:38,062 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a] to archive 2024-12-04T15:11:38,063 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:11:38,066 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/7878fc449f4a44fe851a48d3494268ef 2024-12-04T15:11:38,067 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/b3329fbe4f474becb485a64cbd0f94b0 2024-12-04T15:11:38,068 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/meta/9f6aee7887fa436e9b88887d3f3f3b6a 2024-12-04T15:11:38,071 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/info/8372257e88ed497dadaf5c1f91341425 2024-12-04T15:11:38,074 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ff648bb9ef17:42357 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-04T15:11:38,075 WARN [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [7878fc449f4a44fe851a48d3494268ef=5914, b3329fbe4f474becb485a64cbd0f94b0=5745, 9f6aee7887fa436e9b88887d3f3f3b6a=6646] 2024-12-04T15:11:38,075 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928] to archive 2024-12-04T15:11:38,076 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T15:11:38,078 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/6d6d5ab9ccc944e9a56bb59a9e20bb85 2024-12-04T15:11:38,080 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/00a6b3bf73454805ba3274fdb0479620 2024-12-04T15:11:38,081 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/f3a833403bfc4f00b96e0f5a1f10406e 2024-12-04T15:11:38,082 DEBUG [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928 to hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/archive/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/session/da9cbdfa3d394c85836d56db51641928 2024-12-04T15:11:38,082 WARN [StoreCloser-backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [6d6d5ab9ccc944e9a56bb59a9e20bb85=5049, 00a6b3bf73454805ba3274fdb0479620=5535, f3a833403bfc4f00b96e0f5a1f10406e=6682, da9cbdfa3d394c85836d56db51641928=5928] 2024-12-04T15:11:38,085 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/backup/system/e95ffbd37709d518765ec5c865d8f1ed/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-12-04T15:11:38,086 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:11:38,086 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:38,086 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e95ffbd37709d518765ec5c865d8f1ed: Waiting for close lock at 1733325097512Running coprocessor pre-close hooks at 1733325097512Disabling compacts and flushes for region at 1733325097512Disabling writes for close at 1733325097639 (+127 ms)Obtaining lock to block concurrent updates at 1733325097639Preparing flush snapshotting stores in e95ffbd37709d518765ec5c865d8f1ed at 1733325097639Finished memstore snapshotting backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1733325097640 (+1 ms)Flushing stores of backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. at 1733325097640Flushing e95ffbd37709d518765ec5c865d8f1ed/session: creating writer at 1733325097640Flushing e95ffbd37709d518765ec5c865d8f1ed/session: appending metadata at 1733325097643 (+3 ms)Flushing e95ffbd37709d518765ec5c865d8f1ed/session: closing flushed file at 1733325097643Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c15267: reopening flushed file at 1733325098054 (+411 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for e95ffbd37709d518765ec5c865d8f1ed in 422ms, sequenceid=57, compaction requested=false at 1733325098061 (+7 ms)Writing region close event to WAL at 1733325098083 (+22 ms)Running coprocessor post-close hooks at 1733325098086 (+3 ms)Closed at 1733325098086 2024-12-04T15:11:38,086 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1733325012531.e95ffbd37709d518765ec5c865d8f1ed. 2024-12-04T15:11:38,086 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 031ac2bd7ebf036e19eeefd5230f16ab, disabling compactions & flushes 2024-12-04T15:11:38,086 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:11:38,086 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:11:38,086 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. after waiting 0 ms 2024-12-04T15:11:38,086 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:11:38,089 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/ns4/test-17333250036803/031ac2bd7ebf036e19eeefd5230f16ab/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T15:11:38,089 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:11:38,090 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:11:38,090 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 031ac2bd7ebf036e19eeefd5230f16ab: Waiting for close lock at 1733325098086Running coprocessor pre-close hooks at 1733325098086Disabling compacts and flushes for region at 1733325098086Disabling writes for close at 1733325098086Writing region close event to WAL at 1733325098086Running coprocessor post-close hooks at 1733325098089 (+3 ms)Closed at 1733325098090 (+1 ms) 2024-12-04T15:11:38,090 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17333250036803,,1733325008945.031ac2bd7ebf036e19eeefd5230f16ab. 2024-12-04T15:11:38,090 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 44b146fc21bd97567dc283c76bd9e484, disabling compactions & flushes 2024-12-04T15:11:38,090 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:11:38,090 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:11:38,090 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. after waiting 0 ms 2024-12-04T15:11:38,090 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:11:38,090 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 44b146fc21bd97567dc283c76bd9e484 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-12-04T15:11:38,093 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/ns/91e30e9f2eaa45ffaf54ff97dd274f3f is 43, key is default/ns:d/1733324999255/Put/seqid=0 2024-12-04T15:11:38,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741978_1154 (size=5345) 2024-12-04T15:11:38,097 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/ns/91e30e9f2eaa45ffaf54ff97dd274f3f 2024-12-04T15:11:38,108 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f2/f8c969fa644a49519462ffee2df341a2 is 58, key is row-test-1733325003680-2-0/f2:q1/1733325077252/Put/seqid=0 2024-12-04T15:11:38,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741979_1155 (size=5464) 2024-12-04T15:11:38,116 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/table/2bbedf74dd85407f8b2f63a1d773e7ed is 59, key is ns2:test-17333250036801/table:state/1733325006689/Put/seqid=0 2024-12-04T15:11:38,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741980_1156 (size=5866) 2024-12-04T15:11:38,140 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ff648bb9ef17:42357 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-12-04T15:11:38,243 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:11:38,435 INFO [regionserver/ff648bb9ef17:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T15:11:38,435 INFO [regionserver/ff648bb9ef17:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T15:11:38,444 DEBUG [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 44b146fc21bd97567dc283c76bd9e484 2024-12-04T15:11:38,513 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f2/f8c969fa644a49519462ffee2df341a2 2024-12-04T15:11:38,519 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/.tmp/f2/f8c969fa644a49519462ffee2df341a2 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f2/f8c969fa644a49519462ffee2df341a2 2024-12-04T15:11:38,520 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/table/2bbedf74dd85407f8b2f63a1d773e7ed 2024-12-04T15:11:38,525 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/f2/f8c969fa644a49519462ffee2df341a2, entries=7, sequenceid=348, filesize=5.3 K 2024-12-04T15:11:38,526 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/info/8372257e88ed497dadaf5c1f91341425 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/info/8372257e88ed497dadaf5c1f91341425 2024-12-04T15:11:38,526 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 44b146fc21bd97567dc283c76bd9e484 in 436ms, sequenceid=348, compaction requested=false 2024-12-04T15:11:38,531 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/default/test-1733325003680/44b146fc21bd97567dc283c76bd9e484/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-12-04T15:11:38,531 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:11:38,532 INFO [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:11:38,532 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 44b146fc21bd97567dc283c76bd9e484: Waiting for close lock at 1733325098090Running coprocessor pre-close hooks at 1733325098090Disabling compacts and flushes for region at 1733325098090Disabling writes for close at 1733325098090Obtaining lock to block concurrent updates at 1733325098090Preparing flush snapshotting stores in 44b146fc21bd97567dc283c76bd9e484 at 1733325098090Finished memstore snapshotting test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1733325098090Flushing stores of test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. at 1733325098091 (+1 ms)Flushing 44b146fc21bd97567dc283c76bd9e484/f2: creating writer at 1733325098091Flushing 44b146fc21bd97567dc283c76bd9e484/f2: appending metadata at 1733325098108 (+17 ms)Flushing 44b146fc21bd97567dc283c76bd9e484/f2: closing flushed file at 1733325098108Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f1debec: reopening flushed file at 1733325098518 (+410 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 44b146fc21bd97567dc283c76bd9e484 in 436ms, sequenceid=348, compaction requested=false at 1733325098526 (+8 ms)Writing region close event to WAL at 1733325098528 (+2 ms)Running coprocessor post-close hooks at 1733325098531 (+3 ms)Closed at 1733325098531 2024-12-04T15:11:38,532 DEBUG [RS_CLOSE_REGION-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1733325003680,,1733325004179.44b146fc21bd97567dc283c76bd9e484. 2024-12-04T15:11:38,532 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/info/8372257e88ed497dadaf5c1f91341425, entries=69, sequenceid=59, filesize=13.8 K 2024-12-04T15:11:38,533 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/ns/91e30e9f2eaa45ffaf54ff97dd274f3f as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/ns/91e30e9f2eaa45ffaf54ff97dd274f3f 2024-12-04T15:11:38,538 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/ns/91e30e9f2eaa45ffaf54ff97dd274f3f, entries=7, sequenceid=59, filesize=5.2 K 2024-12-04T15:11:38,539 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/.tmp/table/2bbedf74dd85407f8b2f63a1d773e7ed as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/table/2bbedf74dd85407f8b2f63a1d773e7ed 2024-12-04T15:11:38,543 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/table/2bbedf74dd85407f8b2f63a1d773e7ed, entries=13, sequenceid=59, filesize=5.7 K 2024-12-04T15:11:38,544 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 901ms, sequenceid=59, compaction requested=false 2024-12-04T15:11:38,549 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-12-04T15:11:38,549 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-12-04T15:11:38,549 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T15:11:38,549 INFO [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T15:11:38,549 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733325097642Running coprocessor pre-close hooks at 1733325097642Disabling compacts and flushes for region at 1733325097642Disabling writes for close at 1733325097643 (+1 ms)Obtaining lock to block concurrent updates at 1733325097643Preparing flush snapshotting stores in 1588230740 at 1733325097643Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1733325097643Flushing stores of hbase:meta,,1.1588230740 at 1733325097644 (+1 ms)Flushing 1588230740/info: creating writer at 1733325097644Flushing 1588230740/info: appending metadata at 1733325097666 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733325097666Flushing 1588230740/ns: creating writer at 1733325098076 (+410 ms)Flushing 1588230740/ns: appending metadata at 1733325098092 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733325098092Flushing 1588230740/table: creating writer at 1733325098102 (+10 ms)Flushing 1588230740/table: appending metadata at 1733325098116 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733325098116Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51822134: reopening flushed file at 1733325098525 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8259b21: reopening flushed file at 1733325098532 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b22d9cf: reopening flushed file at 1733325098538 (+6 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 901ms, sequenceid=59, compaction requested=false at 1733325098544 (+6 ms)Writing region close event to WAL at 1733325098546 (+2 ms)Running coprocessor post-close hooks at 1733325098549 (+3 ms)Closed at 1733325098549 2024-12-04T15:11:38,550 DEBUG [RS_CLOSE_META-regionserver/ff648bb9ef17:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T15:11:38,644 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(976): stopping server ff648bb9ef17,41657,1733324995877; all regions closed. 2024-12-04T15:11:38,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741954_1130 (size=3260) 2024-12-04T15:11:38,648 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.meta.1733325086590.meta not finished, retry = 0 2024-12-04T15:11:38,753 DEBUG [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs 2024-12-04T15:11:38,753 INFO [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.meta:.meta(num 1733325086590) 2024-12-04T15:11:38,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741956_1132 (size=5275) 2024-12-04T15:11:38,758 DEBUG [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs 2024-12-04T15:11:38,758 INFO [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-1:(num 1733325086625) 2024-12-04T15:11:38,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741955_1131 (size=1037) 2024-12-04T15:11:38,759 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/WALs/ff648bb9ef17,41657,1733324995877/ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0.1733325086608 not finished, retry = 0 2024-12-04T15:11:38,862 DEBUG [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/oldWALs 2024-12-04T15:11:38,862 INFO [RS:0;ff648bb9ef17:41657 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ff648bb9ef17%2C41657%2C1733324995877.ff648bb9ef17%2C41657%2C1733324995877.regiongroup-0:(num 1733325086608) 2024-12-04T15:11:38,862 DEBUG [RS:0;ff648bb9ef17:41657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:38,862 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T15:11:38,862 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T15:11:38,863 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.ChoreService(370): Chore service for: regionserver/ff648bb9ef17:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T15:11:38,863 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T15:11:38,863 INFO [regionserver/ff648bb9ef17:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T15:11:38,864 INFO [RS:0;ff648bb9ef17:41657 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41657 2024-12-04T15:11:38,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-12-04T15:11:38,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/ff648bb9ef17,41657,1733324995877 2024-12-04T15:11:38,868 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T15:11:38,871 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ff648bb9ef17,41657,1733324995877] 2024-12-04T15:11:38,872 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/ff648bb9ef17,41657,1733324995877 already deleted, retry=false 2024-12-04T15:11:38,873 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ff648bb9ef17,41657,1733324995877 expired; onlineServers=0 2024-12-04T15:11:38,873 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ff648bb9ef17,42357,1733324994229' ***** 2024-12-04T15:11:38,873 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T15:11:38,873 INFO [M:0;ff648bb9ef17:42357 {}] hbase.HBaseServerBase(468): Stop info server 2024-12-04T15:11:38,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-12-04T15:11:38,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-12-04T15:11:38,880 INFO [M:0;ff648bb9ef17:42357 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@e084f9e{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-12-04T15:11:38,880 INFO [M:0;ff648bb9ef17:42357 {}] server.AbstractConnector(383): Stopped ServerConnector@fd624d{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-04T15:11:38,880 INFO [M:0;ff648bb9ef17:42357 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:11:38,881 INFO [M:0;ff648bb9ef17:42357 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4f87c64{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-12-04T15:11:38,881 INFO [M:0;ff648bb9ef17:42357 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@16164b0d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,STOPPED} 2024-12-04T15:11:38,881 INFO [M:0;ff648bb9ef17:42357 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T15:11:38,881 INFO [M:0;ff648bb9ef17:42357 {}] master.HMaster(1770): Stopping master jetty server 2024-12-04T15:11:38,882 INFO [M:0;ff648bb9ef17:42357 {}] server.AbstractConnector(383): Stopped ServerConnector@701715b7{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-12-04T15:11:38,883 INFO [M:0;ff648bb9ef17:42357 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T15:11:38,883 DEBUG [M:0;ff648bb9ef17:42357 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T15:11:38,883 INFO [M:0;ff648bb9ef17:42357 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-12-04T15:11:38,883 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T15:11:38,883 INFO [M:0;ff648bb9ef17:42357 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;ff648bb9ef17:42357. 2024-12-04T15:11:38,883 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster-HFileCleaner.small.0-1733324998281 {}] cleaner.HFileCleaner(306): Exit Thread[master/ff648bb9ef17:0:becomeActiveMaster-HFileCleaner.small.0-1733324998281,5,FailOnTimeoutGroup] 2024-12-04T15:11:38,883 DEBUG [master/ff648bb9ef17:0:becomeActiveMaster-HFileCleaner.large.0-1733324998281 {}] cleaner.HFileCleaner(306): Exit Thread[master/ff648bb9ef17:0:becomeActiveMaster-HFileCleaner.large.0-1733324998281,5,FailOnTimeoutGroup] 2024-12-04T15:11:38,883 DEBUG [M:0;ff648bb9ef17:42357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:11:38,883 DEBUG [M:0;ff648bb9ef17:42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:38,883 DEBUG [M:0;ff648bb9ef17:42357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:11:38,883 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:11:38,883 DEBUG [M:0;ff648bb9ef17:42357 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T15:11:38,884 INFO [M:0;ff648bb9ef17:42357 {}] hbase.ChoreService(370): Chore service for: master/ff648bb9ef17:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T15:11:38,884 INFO [M:0;ff648bb9ef17:42357 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T15:11:38,884 DEBUG [M:0;ff648bb9ef17:42357 {}] master.HMaster(1795): Stopping service threads 2024-12-04T15:11:38,884 INFO [M:0;ff648bb9ef17:42357 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T15:11:38,884 INFO [M:0;ff648bb9ef17:42357 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T15:11:38,885 INFO [M:0;ff648bb9ef17:42357 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T15:11:38,885 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T15:11:38,885 DEBUG [M:0;ff648bb9ef17:42357 {}] zookeeper.ZKUtil(347): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-12-04T15:11:38,885 WARN [M:0;ff648bb9ef17:42357 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T15:11:38,886 INFO [M:0;ff648bb9ef17:42357 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/.lastflushedseqids 2024-12-04T15:11:38,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741981_1157 (size=504) 2024-12-04T15:11:38,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:11:38,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41657-0x1017e065dfa0001, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:11:38,972 INFO [RS:0;ff648bb9ef17:41657 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T15:11:38,972 INFO [RS:0;ff648bb9ef17:41657 {}] regionserver.HRegionServer(1031): Exiting; stopping=ff648bb9ef17,41657,1733324995877; zookeeper connection closed. 2024-12-04T15:11:38,972 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a4c51c9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a4c51c9 2024-12-04T15:11:38,973 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T15:11:39,300 INFO [M:0;ff648bb9ef17:42357 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T15:11:39,300 INFO [M:0;ff648bb9ef17:42357 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T15:11:39,300 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T15:11:39,314 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:11:39,314 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:11:39,314 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T15:11:39,314 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:11:39,314 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=319.26 KB heapSize=375.38 KB 2024-12-04T15:11:39,330 DEBUG [M:0;ff648bb9ef17:42357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/90321c5b9b684a36b892f9cc946d8a43 is 82, key is hbase:meta,,1/info:regioninfo/1733324999188/Put/seqid=0 2024-12-04T15:11:39,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741982_1158 (size=5672) 2024-12-04T15:11:39,386 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-12-04T15:11:39,734 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/90321c5b9b684a36b892f9cc946d8a43 2024-12-04T15:11:39,756 DEBUG [M:0;ff648bb9ef17:42357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e0dbb81665a4dc0aad4e8a5ef23419f is 2184, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1733325048324/Put/seqid=0 2024-12-04T15:11:39,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741983_1159 (size=26519) 2024-12-04T15:11:40,161 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.71 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e0dbb81665a4dc0aad4e8a5ef23419f 2024-12-04T15:11:40,166 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e0dbb81665a4dc0aad4e8a5ef23419f 2024-12-04T15:11:40,181 DEBUG [M:0;ff648bb9ef17:42357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/73755c27d5b847c99d2dbbb063945780 is 69, key is ff648bb9ef17,41657,1733324995877/rs:state/1733324998361/Put/seqid=0 2024-12-04T15:11:40,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741984_1160 (size=5156) 2024-12-04T15:11:40,585 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/73755c27d5b847c99d2dbbb063945780 2024-12-04T15:11:40,590 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/90321c5b9b684a36b892f9cc946d8a43 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/90321c5b9b684a36b892f9cc946d8a43 2024-12-04T15:11:40,595 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/90321c5b9b684a36b892f9cc946d8a43, entries=8, sequenceid=667, filesize=5.5 K 2024-12-04T15:11:40,596 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0e0dbb81665a4dc0aad4e8a5ef23419f as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0e0dbb81665a4dc0aad4e8a5ef23419f 2024-12-04T15:11:40,600 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e0dbb81665a4dc0aad4e8a5ef23419f 2024-12-04T15:11:40,600 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0e0dbb81665a4dc0aad4e8a5ef23419f, entries=69, sequenceid=667, filesize=25.9 K 2024-12-04T15:11:40,601 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/73755c27d5b847c99d2dbbb063945780 as hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/73755c27d5b847c99d2dbbb063945780 2024-12-04T15:11:40,605 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36003/user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/73755c27d5b847c99d2dbbb063945780, entries=1, sequenceid=667, filesize=5.0 K 2024-12-04T15:11:40,606 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(3140): Finished flush of dataSize ~319.26 KB/326923, heapSize ~375.08 KB/384080, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1292ms, sequenceid=667, compaction requested=false 2024-12-04T15:11:40,607 INFO [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:11:40,607 DEBUG [M:0;ff648bb9ef17:42357 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733325099300Disabling compacts and flushes for region at 1733325099300Disabling writes for close at 1733325099314 (+14 ms)Obtaining lock to block concurrent updates at 1733325099314Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733325099314Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326923, getHeapSize=384320, getOffHeapSize=0, getCellsCount=746 at 1733325099315 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733325099315Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733325099315Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733325099329 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733325099329Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733325099740 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733325099755 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733325099755Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733325100166 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733325100180 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733325100180Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14d35ca8: reopening flushed file at 1733325100590 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fdb871f: reopening flushed file at 1733325100595 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e53a3a8: reopening flushed file at 1733325100600 (+5 ms)Finished flush of dataSize ~319.26 KB/326923, heapSize ~375.08 KB/384080, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1292ms, sequenceid=667, compaction requested=false at 1733325100606 (+6 ms)Writing region close event to WAL at 1733325100607 (+1 ms)Closed at 1733325100607 2024-12-04T15:11:40,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38861 is added to blk_1073741830_1006 (size=371013) 2024-12-04T15:11:40,610 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/df849a58-45cb-4eef-368c-3c04b62ebb48/MasterData/WALs/ff648bb9ef17,42357,1733324994229/ff648bb9ef17%2C42357%2C1733324994229.ff648bb9ef17%2C42357%2C1733324994229.regiongroup-0.1733324997584 not finished, retry = 0 2024-12-04T15:11:40,711 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T15:11:40,712 INFO [M:0;ff648bb9ef17:42357 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T15:11:40,712 INFO [M:0;ff648bb9ef17:42357 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-12-04T15:11:40,712 INFO [M:0;ff648bb9ef17:42357 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42357 2024-12-04T15:11:40,712 INFO [M:0;ff648bb9ef17:42357 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T15:11:40,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:11:40,814 INFO [M:0;ff648bb9ef17:42357 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T15:11:40,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42357-0x1017e065dfa0000, quorum=127.0.0.1:50985, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:11:40,821 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79fa0fb2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T15:11:40,823 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@482cd6b5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:11:40,823 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:11:40,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a4f7ff0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T15:11:40,824 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b27bdff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,STOPPED} 2024-12-04T15:11:40,826 WARN [BP-2007428057-172.17.0.2-1733324991063 heartbeating to localhost/127.0.0.1:36003 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T15:11:40,826 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T15:11:40,826 WARN [BP-2007428057-172.17.0.2-1733324991063 heartbeating to localhost/127.0.0.1:36003 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2007428057-172.17.0.2-1733324991063 (Datanode Uuid 5aede1a2-d9bc-4774-9318-20053b7244bc) service to localhost/127.0.0.1:36003 2024-12-04T15:11:40,826 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T15:11:40,828 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/data/data1/current/BP-2007428057-172.17.0.2-1733324991063 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T15:11:40,828 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/cluster_542ca3b0-9e03-e18d-cf00-f3c6f53c0322/data/data2/current/BP-2007428057-172.17.0.2-1733324991063 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T15:11:40,829 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T15:11:40,835 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@41b74231{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T15:11:40,836 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1cfa68d8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:11:40,836 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:11:40,836 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13488ad7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T15:11:40,836 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57594f23{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,STOPPED} 2024-12-04T15:11:40,845 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T15:11:40,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T15:11:40,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-12-04T15:11:40,902 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b3d815c{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-04T15:11:40,903 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@33598bd6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:11:40,903 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:11:40,903 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e8f6b9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-04T15:11:40,903 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59559291{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,STOPPED} 2024-12-04T15:11:40,912 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a9ede50{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-12-04T15:11:40,913 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@27800845{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:11:40,913 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:11:40,913 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69ddd664{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-04T15:11:40,913 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@404f25b1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,STOPPED} 2024-12-04T15:11:40,919 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-04T15:11:40,919 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@db534bf{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-12-04T15:11:40,920 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4aa9facb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:11:40,920 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:11:40,920 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48ed5d50{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-04T15:11:40,920 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@382efc4a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,STOPPED} 2024-12-04T15:11:40,923 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-12-04T15:11:40,928 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-12-04T15:11:40,929 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-12-04T15:11:40,930 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-04T15:11:40,932 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@39087899{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-12-04T15:11:40,933 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@99937f3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:11:40,933 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:11:40,933 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5fbf2bd8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-12-04T15:11:40,933 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e8f084e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/b43b665e-c2a3-02f5-c350-78375a11041a/hadoop.log.dir/,STOPPED} 2024-12-04T15:11:40,935 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-12-04T15:11:40,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped